stats_manager.py

# SPDX-FileCopyrightText: 2020 The Magma Authors.
# SPDX-FileCopyrightText: 2022 Open Networking Foundation <support@opennetworking.org>
#
# SPDX-License-Identifier: BSD-3-Clause

import asyncio
from xml.etree import ElementTree

from aiohttp import web
from common.misc_utils import get_ip_from_if
from configuration.service_configs import load_service_config
from enodeb_status import get_enb_status, update_status_metrics
from logger import EnodebdLogger as logger
from state_machines.enb_acs import EnodebAcsStateMachine
from state_machines.enb_acs_manager import StateMachineManager

import metrics


class StatsManager:
    """ HTTP server to receive performance management uploads from eNodeB and
        translate to metrics """
    # Dict to map performance counter names (from eNodeB) to metrics
    # For eNodeB sub-counters, the counter name is shown as
    # '<counter>:<sub-counter>'
    PM_FILE_TO_METRIC_MAP = {
        'RRC.AttConnEstab': metrics.STAT_RRC_ESTAB_ATT,
        'RRC.SuccConnEstab': metrics.STAT_RRC_ESTAB_SUCC,
        'RRC.AttConnReestab': metrics.STAT_RRC_REESTAB_ATT,
        'RRC.AttConnReestab._Cause:RRC.AttConnReestab.RECONF_FAIL':
            metrics.STAT_RRC_REESTAB_ATT_RECONF_FAIL,
        'RRC.AttConnReestab._Cause:RRC.AttConnReestab.HO_FAIL':
            metrics.STAT_RRC_REESTAB_ATT_HO_FAIL,
        'RRC.AttConnReestab._Cause:RRC.AttConnReestab.OTHER':
            metrics.STAT_RRC_REESTAB_ATT_OTHER,
        'RRC.SuccConnReestab': metrics.STAT_RRC_REESTAB_SUCC,
        'ERAB.NbrAttEstab': metrics.STAT_ERAB_ESTAB_ATT,
        'ERAB.NbrSuccEstab': metrics.STAT_ERAB_ESTAB_SUCC,
        'ERAB.NbrFailEstab': metrics.STAT_ERAB_ESTAB_FAIL,
        'ERAB.NbrReqRelEnb': metrics.STAT_ERAB_REL_REQ,
        'ERAB.NbrReqRelEnb.CauseUserInactivity':
            metrics.STAT_ERAB_REL_REQ_USER_INAC,
        'ERAB.NbrReqRelEnb.Normal': metrics.STAT_ERAB_REL_REQ_NORMAL,
        'ERAB.NbrReqRelEnb._Cause:ERAB.NbrReqRelEnb.CauseRADIORESOURCESNOTAVAILABLE':
            metrics.STAT_ERAB_REL_REQ_RES_NOT_AVAIL,
        'ERAB.NbrReqRelEnb._Cause:ERAB.NbrReqRelEnb.CauseREDUCELOADINSERVINGCELL':
            metrics.STAT_ERAB_REL_REQ_REDUCE_LOAD,
        'ERAB.NbrReqRelEnb._Cause:ERAB.NbrReqRelEnb.CauseFAILUREINTHERADIOINTERFACEPROCEDURE':
            metrics.STAT_ERAB_REL_REQ_FAIL_IN_RADIO,
        'ERAB.NbrReqRelEnb._Cause:ERAB.NbrReqRelEnb.CauseRELEASEDUETOEUTRANGENERATEDREASONS':
            metrics.STAT_ERAB_REL_REQ_EUTRAN_REAS,
        'ERAB.NbrReqRelEnb._Cause:ERAB.NbrReqRelEnb.CauseRADIOCONNECTIONWITHUELOST':
            metrics.STAT_ERAB_REL_REQ_RADIO_CONN_LOST,
        'ERAB.NbrReqRelEnb._Cause:ERAB.NbrReqRelEnb.CauseOAMINTERVENTION':
            metrics.STAT_ERAB_REL_REQ_OAM_INTV,
        'PDCP.UpOctUl': metrics.STAT_PDCP_USER_PLANE_BYTES_UL,
        'PDCP.UpOctDl': metrics.STAT_PDCP_USER_PLANE_BYTES_DL,
    }

    # Check if radio transmit is turned on every 10 seconds.
    CHECK_RF_TX_PERIOD = 10

    def __init__(self, enb_acs_manager: StateMachineManager):
        self.enb_manager = enb_acs_manager
        self.loop = asyncio.get_event_loop()
        self._prev_rf_tx = False
        self.mme_timeout_handler = None

    def run(self) -> None:
        """ Create and start HTTP server """
        svc_config = load_service_config("enodebd")

        app = web.Application()
        app.router.add_route(
            'POST', "/{something}",
            self._post_and_put_handler,
        )
        app.router.add_route(
            'PUT', "/{something}",
            self._post_and_put_handler,
        )

        handler = app.make_handler()
        create_server_func = self.loop.create_server(
            handler,
            host=get_ip_from_if(svc_config['tr069']['interface']),
            port=svc_config['tr069']['perf_mgmt_port'],
        )

        self._periodic_check_rf_tx()
        self.loop.run_until_complete(create_server_func)

    def _periodic_check_rf_tx(self) -> None:
        self._check_rf_tx()
        self.mme_timeout_handler = self.loop.call_later(
            self.CHECK_RF_TX_PERIOD,
            self._periodic_check_rf_tx,
        )

    def _check_rf_tx(self) -> None:
        """
        Check if eNodeB should be connected to MME but isn't, and maybe reboot.

        If the eNB doesn't report connection to MME within a timeout period,
        get it to reboot in the hope that it will fix things.

        Usually, enodebd polls the eNodeB for whether it is connected to MME.
        This method checks the last polled MME connection status, and if
        eNodeB should be connected to MME but it isn't.
        """
        # Clear stats when eNodeB stops radiating. This is
        # because eNodeB stops sending performance metrics at this point.
        serial_list = self.enb_manager.get_connected_serial_id_list()
        for enb_serial in serial_list:
            handler = self.enb_manager.get_handler_by_serial(enb_serial)
            if handler:
                self._check_rf_tx_for_handler(handler)

    def _check_rf_tx_for_handler(self, handler: EnodebAcsStateMachine) -> None:
        status = get_enb_status(handler)
        if self._prev_rf_tx and not status.rf_tx_on:
            self._clear_stats()
        self._prev_rf_tx = status.rf_tx_on

        # Update status metrics
        update_status_metrics(status)

    def _get_enb_label_from_request(self, request) -> str:
        label = 'default'
        ip = request.headers.get('X-Forwarded-For')

        if ip is None:
            peername = request.transport.get_extra_info('peername')
            if peername is not None:
                ip, _ = peername

        if ip is None:
            return label

        label = self.enb_manager.get_serial_of_ip(ip)
        if label:
            logger.debug('Found serial %s for ip %s', label, ip)
        else:
            logger.error("Couldn't find serial for ip", ip)
        return label

    @asyncio.coroutine
    def _post_and_put_handler(self, request) -> web.Response:
        """ HTTP POST handler """
        # Read request body and convert to XML tree
        body = yield from request.read()

        root = ElementTree.fromstring(body)
        label = self._get_enb_label_from_request(request)
        if label:
            self._parse_pm_xml(label, root)

        # Return success response
        return web.Response()

    def _parse_pm_xml(self, enb_label, xml_root) -> None:
        """
        Parse performance management XML from eNodeB and populate metrics.
        The schema for this XML document, along with an example, is shown in
        tests/stats_manager_tests.py.
        """
        for measurement in xml_root.findall('Measurements'):
            object_type = measurement.findtext('ObjectType')
            names = measurement.find('PmName')
            data = measurement.find('PmData')
            if object_type == 'EutranCellTdd':
                self._parse_tdd_counters(enb_label, names, data)
            elif object_type == 'ManagedElement':
                # Currently no counters to parse
                pass
            elif object_type == 'SctpAssoc':
                # Currently no counters to parse
                pass

    def _parse_tdd_counters(self, enb_label, names, data):
        """
        Parse eNodeB performance management counters from TDD structure.
        Most of the logic is just to extract the correct counter based on the
        name of the statistic. Each counter is either of type 'V', which is a
        single integer value, or 'CV', which contains multiple integer
        sub-elements, named 'SV', which we add together. E.g:
        <V i="9">0</V>
        <CV i="10">
          <SN>RRC.AttConnReestab.RECONF_FAIL</SN>
          <SV>0</SV>
          <SN>RRC.AttConnReestab.HO_FAIL</SN>
          <SV>0</SV>
          <SN>RRC.AttConnReestab.OTHER</SN>
          <SV>0</SV>
        </CV>
        See tests/stats_manager_tests.py for a more complete example.
        """
        index_data_map = self._build_index_to_data_map(data)
        name_index_map = self._build_name_to_index_map(names)

        # For each performance metric, extract value from XML document and set
        # internal metric to that value.
        for pm_name, metric in self.PM_FILE_TO_METRIC_MAP.items():

            elements = pm_name.split(':')
            counter = elements.pop(0)
            if len(elements) == 0:
                subcounter = None
            else:
                subcounter = elements.pop(0)

            index = name_index_map.get(counter)
            if index is None:
                logger.warning('PM counter %s not found in PmNames', counter)
                continue

            data_el = index_data_map.get(index)
            if data_el is None:
                logger.warning('PM counter %s not found in PmData', counter)
                continue

            if data_el.tag == 'V':
                if subcounter is not None:
                    logger.warning('No subcounter in PM counter %s', counter)
                    continue

                # Data is singular value
                try:
                    value = int(data_el.text)
                except ValueError:
                    logger.info(
                        'PM value (%s) of counter %s not integer',
                        data_el.text, counter,
                    )
                    continue
            elif data_el.tag == 'CV':
                # Check whether we want just one subcounter, or sum them all
                subcounter_index = None
                if subcounter is not None:
                    index = 0
                    for sub_name_el in data_el.findall('SN'):
                        if sub_name_el.text == subcounter:
                            subcounter_index = index
                        index = index + 1

                if subcounter is not None and subcounter_index is None:
                    logger.warning('PM subcounter (%s) not found', subcounter)
                    continue

                # Data is multiple sub-elements. Sum them, or select the one
                # of interest
                value = 0
                try:
                    index = 0
                    for sub_data_el in data_el.findall('SV'):
                        if subcounter_index is None or \
                                subcounter_index == index:
                            value = value + int(sub_data_el.text)
                        index = index + 1
                except ValueError:
                    logger.error(
                        'PM value (%s) of counter %s not integer',
                        sub_data_el.text, pm_name,
                    )
                    continue
            else:
                logger.warning(
                    'Unknown PM data type (%s) of counter %s',
                    data_el.tag, pm_name,
                )
                continue

            # Apply new value to metric
            if pm_name == 'PDCP.UpOctUl' or pm_name == 'PDCP.UpOctDl':
                metric.labels(enb_label).set(value)
            else:
                metric.set(value)

    def _build_index_to_data_map(self, data_etree):
        """
        Parse XML ElementTree and build a dict mapping index to data XML
        element. The relevant part of XML schema being parsed is:
        <xs:element name="PmData">
         <xs:complexType>
          <xs:sequence minOccurs="0" maxOccurs="unbounded">
           <xs:element name="Pm">
            <xs:complexType>
             <xs:choice minOccurs="0" maxOccurs="unbounded">
              <xs:element name="V">
               <xs:complexType>
                <xs:simpleContent>
                 <xs:extension base="xs:string">
                  <xs:attribute name="i" type="xs:integer" use="required"/>
                 </xs:extension>
                </xs:simpleContent>
               </xs:complexType>
              </xs:element>
              <xs:element name="CV">
               <xs:complexType>
                <xs:sequence minOccurs="0" maxOccurs="unbounded">
                 <xs:element name="SN" type="xs:string"/>
                 <xs:element name="SV" type="xs:string"/>
                </xs:sequence>
                <xs:attribute name="i" type="xs:integer" use="required"/>
               </xs:complexType>
              </xs:element>
             </xs:choice>
             <xs:attribute name="Dn" type="xs:string" use="required"/>
             <xs:attribute name="UserLabel" type="xs:string" use="required"/>
            </xs:complexType>
           </xs:element>
          </xs:sequence>
         </xs:complexType>
        </xs:element>

        Inputs:
            - XML elementree element corresponding to 'PmData' in above schema
        Outputs:
            - Dict mapping index ('i' in above schema) to data elementree
              elements ('V' and 'CV' in above schema)
        """
        # Construct map of index to pm_data XML element
        index_data_map = {}
        for pm_el in data_etree.findall('Pm'):
            for data_el in pm_el.findall('V'):
                index = data_el.get('i')
                if index is not None:
                    index_data_map[index] = data_el
            for data_el in pm_el.findall('CV'):
                index = data_el.get('i')
                if index is not None:
                    index_data_map[index] = data_el

        return index_data_map

    def _build_name_to_index_map(self, name_etree):
        """
        Parse XML ElementTree and build a dict mapping name to index. The
        relevant part of XML schema being parsed is:
        <xs:element name="PmName">
         <xs:complexType>
          <xs:sequence minOccurs="0" maxOccurs="unbounded">
           <xs:element name="N">
            <xs:complexType>
             <xs:simpleContent>
              <xs:extension base="xs:string">
               <xs:attribute name="i" type="xs:integer" use="required"/>
              </xs:extension>
             </xs:simpleContent>
            </xs:complexType>
           </xs:element>
          </xs:sequence>
         </xs:complexType>
        </xs:element>

        Inputs:
            - XML elementree element corresponding to 'PmName' in above schema
        Outputs:
            - Dict mapping name ('N' in above schema) to index ('i' in above
              schema)
        """
        # Construct map of pm_name to index
        name_index_map = {}
        for name_el in name_etree.findall('N'):
            name_index_map[name_el.text] = name_el.get('i')

        return name_index_map

    def _clear_stats(self) -> None:
        """
        Clear statistics. Called when eNodeB management plane disconnects
        """
        logger.info('Clearing performance counter statistics')
        # Set all metrics to 0 if eNodeB not connected
        for pm_name, metric in self.PM_FILE_TO_METRIC_MAP.items():
            # eNB data usage metrics will not be cleared
            if pm_name not in ('PDCP.UpOctUl', 'PDCP.UpOctDl'):
                metric.set(0)