| #!/usr/bin/env python |
| |
| # Copyright 2020-present Open Networking Foundation |
| # |
| # SPDX-License-Identifier: Apache-2.0 |
| |
| import os |
| import time |
| import datetime |
| import pytz |
| import threading |
| from icalevents.icalevents import events |
| from flask import Flask, jsonify, abort, request, Response |
| import prometheus_client as prom |
| import jsonschema |
| from logging.config import dictConfig |
| |
| # URL of maintenance calendar |
| SECRET_ICAL_URL = os.environ.get("SECRET_ICAL_URL") |
| |
| # Aether environment that the server is monitoring (e.g., "production") |
| # To schedule downtime, postfix the cluster name with the env: "ace-tucson-production" |
| AETHER_ENV = os.environ.get("AETHER_ENV", "production") |
| |
| # Move to "no result" status if we don't hear from agent for this many seconds |
| NO_RESULT_THRESHOLD = 360 |
| |
| dictConfig({ |
| 'version': 1, |
| 'formatters': {'default': { |
| 'format': '%(levelname)s %(message)s', |
| }}, |
| 'handlers': {'wsgi': { |
| 'class': 'logging.StreamHandler', |
| 'stream': 'ext://flask.logging.wsgi_errors_stream', |
| 'formatter': 'default' |
| }}, |
| 'root': { |
| 'level': 'INFO', |
| 'handlers': ['wsgi'] |
| } |
| }) |
| |
| app = Flask(__name__) |
| |
| edgeSchema = { |
| "type": "object", |
| "properties": { |
| "name": {"type": "string"}, |
| "status": { |
| "type": "object", |
| "properties": { |
| "control_plane": {"type": "string"}, |
| "user_plane": {"type": "string"} |
| }, |
| "required": ["control_plane", "user_plane"] |
| }, |
| "dongle_stats": { |
| "type": "object", |
| "properties": { |
| "SuccessfulFetch": {"type": "boolean"}, |
| "MAC": {"type": "string"}, |
| "PLMNStatus": {"type": "string"}, |
| "UICCStatus": {"type": "string"}, |
| "IMEI": {"type": "string"}, |
| "IMSI": {"type": "string"}, |
| "PLMNSelected": {"type": "string"}, |
| "MCC": {"type": "string"}, |
| "MNC": {"type": "string"}, |
| "PhyCellID": {"type": "string"}, |
| "CellGlobalID": {"type": "string"}, |
| "Band": {"type": "string"}, |
| "EARFCN": {"type": "string"}, |
| "BandWidth": {"type": "string"}, |
| "ServCellState": {"type": "string"}, |
| "Connection": {"type": "string"}, |
| "IPv4Addr": {"type": "string"} |
| } |
| }, |
| "speedtest": { |
| "type": "object", |
| "properties": { |
| "ping": { |
| "type": "object", |
| "properties": { |
| "dry_run": { |
| "type": "object", |
| "properties": { |
| "transmitted": {"type": "number"}, |
| "received": {"type": "number"}, |
| "median": {"type": "number"}, |
| "min": {"type": "number"}, |
| "avg": {"type": "number"}, |
| "max": {"type": "number"}, |
| "stddev": {"type": "number"} |
| }, |
| "required": ["min", "avg", "max", "stddev"] |
| }, |
| "dns": { |
| "type": "object", |
| "properties": { |
| "transmitted": {"type": "number"}, |
| "received": {"type": "number"}, |
| "median": {"type": "number"}, |
| "min": {"type": "number"}, |
| "avg": {"type": "number"}, |
| "max": {"type": "number"}, |
| "stddev": {"type": "number"} |
| }, |
| "required": ["min", "avg", "max", "stddev"] |
| }, |
| "iperf_server": { |
| "type": "object", |
| "properties": { |
| "transmitted": {"type": "number"}, |
| "received": {"type": "number"}, |
| "median": {"type": "number"}, |
| "min": {"type": "number"}, |
| "avg": {"type": "number"}, |
| "max": {"type": "number"}, |
| "stddev": {"type": "number"} |
| }, |
| "required": ["min", "avg", "max", "stddev"] |
| }, |
| "management_server": { |
| "type": "object", |
| "properties": { |
| "transmitted": {"type": "number"}, |
| "received": {"type": "number"}, |
| "median": {"type": "number"}, |
| "min": {"type": "number"}, |
| "avg": {"type": "number"}, |
| "max": {"type": "number"}, |
| "stddev": {"type": "number"} |
| }, |
| "required": ["min", "avg", "max", "stddev"] |
| }, |
| } |
| }, |
| "iperf": { |
| "type": "object", |
| "properties": { |
| "cluster": { |
| "type": "object", |
| "properties": { |
| "downlink": {"type": "number"}, |
| "uplink": {"type": "number"} |
| }, |
| "required": ["downlink", "uplink"] |
| } |
| } |
| } |
| } |
| }, |
| "signal_quality": { |
| "type": "object", |
| "properties": { |
| "rsrq": {"type": "number"}, |
| "rsrp": {"type": "number"} |
| }, |
| "required": ["rsrq", "rsrp"] |
| }, |
| "counters": { |
| "type": "object", |
| "properties": { |
| 'dongle_read_error': {"type": "number"}, |
| 'dongle_connect_error': {"type": "number"}, |
| 'dongle_rsrp_rsrq_error': {"type": "number"}, |
| 'modem_cfun0_error': {"type": "number"}, |
| 'modem_cfun1_error': {"type": "number"}, |
| 'modem_cgatt_error': {"type": "number"}, |
| 'modem_cesq_error': {"type": "number"}, |
| 'dry_run_ping_error': {"type": "number"}, |
| 'ping_error': {"type": "number"}, |
| 'iperf_error': {"type": "number"}, |
| 'report_send_error': {"type": "number"} |
| } |
| } |
| }, |
| "required": ["name", "status"] |
| } |
| |
| edges = [ |
| { |
| 'name': 'ace-example', |
| 'status': { |
| 'control_plane': 'connected', |
| 'user_plane': 'connected' |
| }, |
| 'dongle_stats': { |
| 'SuccessfulFetch': False |
| }, |
| 'speedtest': { |
| 'ping': { |
| 'dry_run': { |
| 'transmitted': 0, |
| 'received': 0, |
| 'median': 0.0, |
| 'min': 0.0, |
| 'avg': 0.0, |
| 'max': 0.0, |
| 'stddev': 0.0 |
| }, |
| 'dns': { |
| 'transmitted': 0, |
| 'received': 0, |
| 'median': 0.0, |
| 'min': 0.0, |
| 'avg': 0.0, |
| 'max': 0.0, |
| 'stddev': 0.0 |
| }, |
| 'iperf_server': { |
| 'transmitted': 0, |
| 'received': 0, |
| 'median': 0.0, |
| 'min': 0.0, |
| 'avg': 0.0, |
| 'max': 0.0, |
| 'stddev': 0.0 |
| }, |
| 'management_server': { |
| 'transmitted': 0, |
| 'received': 0, |
| 'median': 0.0, |
| 'min': 0.0, |
| 'avg': 0.0, |
| 'max': 0.0, |
| 'stddev': 0.0 |
| } |
| }, |
| 'iperf': { |
| 'cluster': { |
| 'downlink': 0.0, |
| 'uplink': 0.0 |
| } |
| } |
| }, |
| 'signal_quality': { |
| 'rsrq': 0, |
| 'rsrp': 0 |
| }, |
| 'counters': { |
| 'dongle_read_error': 0, |
| 'dongle_connect_error': 0, |
| 'dongle_rsrp_rsrq_error': 0, |
| 'modem_cfun0_error': 0, |
| 'modem_cfun1_error': 0, |
| 'modem_cgatt_error': 0, |
| 'modem_cesq_error': 0, |
| 'dry_run_ping_error': 0, |
| 'ping_error': 0, |
| 'iperf_error': 0, |
| 'report_send_error': 0 |
| }, |
| 'last_update': time.time() |
| } |
| ] |
| |
| status_codes = { |
| "no result": -2, |
| "error": -1, |
| "disconnected": 0, |
| "connecting": 1, |
| "connected": 2 |
| } |
| |
| room_mapping = { |
| "ace-menlo-rasp-pi-production": "(Compute)-MP-1-Aether Production", |
| "ace-menlo-staging": "(Compute)-MP-1-Aether Staging" |
| } |
| |
| plmnstatus_mapping = { |
| "Not Searching": 1, |
| "Searching": 2, |
| "Success": 3 |
| } |
| |
| # Legacy test status metrics, reporting a status code between -2 and 2 |
| cp_status = prom.Gauge("aetheredge_status_control_plane", "Control plane status code", ["name"]) |
| up_status = prom.Gauge("aetheredge_status_user_plane", "User plane status code", ["name"]) |
| |
| # Simplified binary test result metrics |
| e2e_tests_ok = prom.Gauge("aetheredge_e2e_tests_ok", "Last connect and ping test both passed", ["name"]) |
| connect_test_ok = prom.Gauge("aetheredge_connect_test_ok", "Last connect test passed", ["name"]) |
| ping_test_ok = prom.Gauge("aetheredge_ping_test_ok", "Last ping test passed", ["name"]) |
| e2e_tests_down = prom.Gauge("aetheredge_e2e_tests_down", "E2E tests not reporting", ["name"]) |
| |
| # Dongle Stats (strings can't be stored in Prometheus) |
| dongle_stats_imsi = prom.Gauge("aetheredge_dongle_stats_imsi", "IMSI of the UE", ["name"]) |
| dongle_stats_cellglobalid = prom.Gauge("aetheredge_dongle_stats_cellglobalid", "CellGlobalID of the UE", ["name"]) |
| dongle_stats_plmnstatus = prom.Gauge("aetheredge_dongle_stats_plmnstatus", "PLMNStatus of the UE", ["name"]) |
| dongle_stats_phycellid = prom.Gauge("aetheredge_dongle_stats_phycellid", "PhyCellID of the UE", ["name"]) |
| |
| # Ping dry_run metrics |
| ping_dry_run_transmitted = prom.Gauge("aetheredge_ping_dry_run_test_transmitted", "Last ping test to dry_run number of transmitted packets", ["name"]) |
| ping_dry_run_received = prom.Gauge("aetheredge_ping_dry_run_test_received", "Last ping test to dry_run number of received packets",["name"]) |
| ping_dry_run_median = prom.Gauge("aetheredge_ping_dry_run_test_median", "Last ping test to dry_run median value", ["name"]) |
| ping_dry_run_min = prom.Gauge("aetheredge_ping_dry_run_test_min", "Last ping test to dry_run minimum value", ["name"]) |
| ping_dry_run_avg = prom.Gauge("aetheredge_ping_dry_run_test_avg", "Last ping test to dry_run average", ["name"]) |
| ping_dry_run_max = prom.Gauge("aetheredge_ping_dry_run_test_max", "Last ping test to dry_run maximum value", ["name"]) |
| ping_dry_run_stddev = prom.Gauge("aetheredge_ping_dry_run_test_stddev", "Last ping test to dry_run standard deviation", ["name"]) |
| |
| # Ping dns metrics |
| ping_dns_transmitted = prom.Gauge("aetheredge_ping_dns_test_transmitted", "Last ping test to dns number of transmitted packets", ["name"]) |
| ping_dns_received = prom.Gauge("aetheredge_ping_dns_test_received", "Last ping test to dns number of received packets", ["name"]) |
| ping_dns_median = prom.Gauge("aetheredge_ping_dns_test_median", "Last ping test to dns median value",["name"]) |
| ping_dns_min = prom.Gauge("aetheredge_ping_dns_test_min", "Last ping test to dns minimum value",["name"]) |
| ping_dns_avg = prom.Gauge("aetheredge_ping_dns_test_avg", "Last ping test to dns average",["name"]) |
| ping_dns_max = prom.Gauge("aetheredge_ping_dns_test_max", "Last ping test to dns maximum value",["name"]) |
| ping_dns_stddev = prom.Gauge("aetheredge_ping_dns_test_stddev", "Last ping test to dns standard deviation",["name"]) |
| |
| # Ping iperf server metrics |
| ping_iperf_server_transmitted = prom.Gauge("aetheredge_ping_iperf_server_test_transmitted","Last ping test to iperf_server number of transmitted packets", ["name"]) |
| ping_iperf_server_received = prom.Gauge("aetheredge_ping_iperf_server_test_received","Last ping test to iperf_server number of received packets", ["name"]) |
| ping_iperf_server_median = prom.Gauge("aetheredge_ping_iperf_server_test_median","Last ping test to iperf_server median value", ["name"]) |
| ping_iperf_server_min = prom.Gauge("aetheredge_ping_iperf_server_test_min","Last ping test to iperf_server minimum value", ["name"]) |
| ping_iperf_server_avg = prom.Gauge("aetheredge_ping_iperf_server_test_avg","Last ping test to iperf_server average", ["name"]) |
| ping_iperf_server_max = prom.Gauge("aetheredge_ping_iperf_server_test_max","Last ping test to iperf_server maximum value", ["name"]) |
| ping_iperf_server_stddev = prom.Gauge("aetheredge_ping_iperf_server_test_stddev","Last ping test to iperf_server standard deviation", ["name"]) |
| |
| # Ping management_server metrics |
| ping_management_server_transmitted = prom.Gauge("aetheredge_ping_management_server_test_transmitted","Last ping test to management_server number of transmitted packets", ["name"]) |
| ping_management_server_received = prom.Gauge("aetheredge_ping_management_server_test_received","Last ping test to management_server number of received packets", ["name"]) |
| ping_management_server_median = prom.Gauge("aetheredge_ping_management_server_test_median","Last ping test to management_server median value", ["name"]) |
| ping_management_server_min = prom.Gauge("aetheredge_ping_management_server_test_min","Last ping test to management_server minimum value",["name"]) |
| ping_management_server_avg = prom.Gauge("aetheredge_ping_management_server_test_avg","Last ping test to management_server average", ["name"]) |
| ping_management_server_max = prom.Gauge("aetheredge_ping_management_server_test_max","Last ping test to management_server maximum value", ["name"]) |
| ping_management_server_stddev = prom.Gauge("aetheredge_ping_management_server_test_stddev","Last ping test to management_server standard deviation", ["name"]) |
| |
| # Speedtest iperf metrics |
| iperf_cluster_downlink = prom.Gauge("aetheredge_iperf_cluster_downlink_test","Last iperf test downlink result",["name"]) |
| iperf_cluster_uplink = prom.Gauge("aetheredge_iperf_cluster_uplink_test","Last iperf test downlink result",["name"]) |
| |
| # Signal quality metrics in CESQ format not dB |
| # RSRQ: >=53 excellent, 43 ~ 53 good, 33 ~ 43 mid, <=33 bad, 0 no signal |
| # RSRP: >=20 excellent, 10 ~ 20 good, 0 ~ 10 mid, 0 no signal |
| signal_quality_rsrq = prom.Gauge("aetheredge_signal_quality_rsrq", "Quality of the received signal", ["name"]) |
| signal_quality_rsrp = prom.Gauge("aetheredge_signal_quality_rsrp", "Power of the received signal", ["name"]) |
| |
| counters_dongle_read_error = prom.Gauge("aetheredge_counters_dongle_read_error", "Dongle read error counter", ["name"]) |
| counters_dongle_connect_error = prom.Gauge("aetheredge_counters_dongle_connect_error", "Dongle connect error counter", ["name"]) |
| counters_dongle_rsrp_rsrq_error = prom.Gauge("aetheredge_counters_dongle_rsrp_rsrq_error", "Dongle rsrp/rsrq error counter", ["name"]) |
| counters_modem_cfun0_error = prom.Gauge("aetheredge_counters_modem_cfun0_error", "Modem cfun0 error counter", ["name"]) |
| counters_modem_cfun1_error = prom.Gauge("aetheredge_counters_modem_cfun1_error", "Modem cfun1 error counter", ["name"]) |
| counters_modem_cgatt_error = prom.Gauge("aetheredge_counters_modem_cgatt_error", "Modem cgatt error counter", ["name"]) |
| counters_modem_cesq_error = prom.Gauge("aetheredge_counters_modem_cesq_error", "Modem cesq error counter", ["name"]) |
| counters_dry_run_ping_error = prom.Gauge("aetheredge_counters_dry_run_ping_error", "Dry run ping error counter", ["name"]) |
| counters_ping_error = prom.Gauge("aetheredge_counters_ping_error", "Ping error counter", ["name"]) |
| counters_iperf_error = prom.Gauge("aetheredge_counters_iperf_error", "Iperf error counter", ["name"]) |
| counters_report_send_error = prom.Gauge("aetheredge_counters_report_send_error", "Report send error counter", ["name"]) |
| |
| # Other metrics |
| last_update = prom.Gauge("aetheredge_last_update", "Last reported test result", ["name"]) |
| maint_window = prom.Gauge("aetheredge_in_maintenance_window", "Currently in a maintenance window", ["name"]) |
| |
| def is_my_event(event, name): |
| for field in ["summary", "location", "description"]: |
| fullname = name |
| if name.startswith("ace-"): |
| fullname = "%s-%s" % (name, AETHER_ENV) |
| if fullname in getattr(event, field, ""): |
| return True |
| if fullname in room_mapping and room_mapping[fullname] in getattr(event, field, ""): |
| return True |
| return False |
| |
| def is_naive_datetime(d): |
| return d.tzinfo is None or d.tzinfo.utcoffset(d) is None |
| |
| def process_all_day_events(es): |
| for event in es: |
| if event.all_day: |
| # All day events have naive datetimes, which breaks comparisons |
| pacific = pytz.timezone('US/Pacific') |
| if is_naive_datetime(event.start): |
| event.start = pacific.localize(event.start) |
| if is_naive_datetime(event.end): |
| event.end = pacific.localize(event.end) |
| |
| def in_maintenance_window(events, name, now): |
| for event in events: |
| if event.start < now and event.end > now: |
| if is_my_event(event, name): |
| return True |
| return False |
| |
| def pull_maintenance_events(): |
| while(True): |
| now = datetime.datetime.now(pytz.utc) |
| try: |
| es = events(SECRET_ICAL_URL, start = now) |
| process_all_day_events(es) |
| except Exception as e: |
| app.logger.error(e) |
| else: |
| for edge in edges: |
| if 'maintenance' not in edge: |
| edge['maintenance'] = {} |
| edge['maintenance']['in_window'] = in_maintenance_window(es, edge['name'], now) |
| edge['maintenance']['last_update'] = time.time() |
| time.sleep(60) |
| |
| def time_out_stale_results(): |
| for edge in edges: |
| time_elapsed = time.time() - edge["last_update"] |
| if time_elapsed > NO_RESULT_THRESHOLD: |
| edge['status']['control_plane'] = "no result" |
| edge['status']['user_plane'] = "no result" |
| edge['dongle_stats'] = {'SuccessfulFetch': False} |
| edge['speedtest']['ping']['dry_run'] = {'transmitted': 0, |
| 'received': 0, |
| 'median': 0.0, |
| 'min': 0.0, |
| 'avg': 0.0, |
| 'max': 0.0, |
| 'stddev': 0.0} |
| edge['speedtest']['ping']['dns'] = {'transmitted': 0, |
| 'received': 0, |
| 'median': 0.0, |
| 'min': 0.0, |
| 'avg': 0.0, |
| 'max': 0.0, |
| 'stddev': 0.0} |
| edge['speedtest']['ping']['iperf_server'] = {'transmitted': 0, |
| 'received': 0, |
| 'median': 0.0, |
| 'min': 0.0, |
| 'avg': 0.0, |
| 'max': 0.0, |
| 'stddev': 0.0} |
| edge['speedtest']['ping']['management_server'] = {'transmitted': 0, |
| 'received': 0, |
| 'median': 0.0, |
| 'min': 0.0, |
| 'avg': 0.0, |
| 'max': 0.0, |
| 'stddev': 0.0} |
| edge['speedtest']['iperf'] = {'cluster': { |
| 'downlink': 0.0, |
| 'uplink': 0.0 |
| } |
| } |
| edge.pop('signal_quality', None) |
| |
| def remove_edge_from_metrics(name): |
| try: |
| cp_status.remove(name) |
| up_status.remove(name) |
| last_update.remove(name) |
| e2e_tests_ok.remove(name) |
| connect_test_ok.remove(name) |
| ping_test_ok.remove(name) |
| e2e_tests_down.remove(name) |
| except: |
| pass |
| |
| try: |
| dongle_stats_imsi.remove(name) |
| dongle_stats_cellglobalid.remove(name) |
| dongle_stats_plmnstatus.remove(name) |
| dongle_stats_phycellid.remove(name) |
| except: |
| pass |
| |
| try: |
| ping_dry_run_min.remove(name) |
| ping_dry_run_avg.remove(name) |
| ping_dry_run_max.remove(name) |
| ping_dry_run_stddev.remove(name) |
| ping_dry_run_transmitted.remove(name) |
| ping_dry_run_received.remove(name) |
| ping_dry_run_median.remove(name) |
| except: |
| pass |
| |
| try: |
| ping_dns_min.remove(name) |
| ping_dns_avg.remove(name) |
| ping_dns_max.remove(name) |
| ping_dns_stddev.remove(name) |
| ping_dns_transmitted.remove(name) |
| ping_dns_received.remove(name) |
| ping_dns_median.remove(name) |
| except: |
| pass |
| |
| try: |
| ping_iperf_server_min.remove(name) |
| ping_iperf_server_avg.remove(name) |
| ping_iperf_server_max.remove(name) |
| ping_iperf_server_stddev.remove(name) |
| ping_iperf_server_transmitted.remove(name) |
| ping_iperf_server_received.remove(name) |
| ping_iperf_server_median.remove(name) |
| except: |
| pass |
| |
| try: |
| ping_management_server_min.remove(name) |
| ping_management_server_avg.remove(name) |
| ping_management_server_max.remove(name) |
| ping_management_server_stddev.remove(name) |
| ping_management_server_transmitted.remove(name) |
| ping_management_server_received.remove(name) |
| ping_management_server_median.remove(name) |
| except: |
| pass |
| |
| try: |
| iperf_cluster_downlink.remove(name) |
| iperf_cluster_uplink.remove(name) |
| except: |
| pass |
| |
| try: |
| signal_quality_rsrq.remove(name) |
| signal_quality_rsrp.remove(name) |
| except: |
| pass |
| |
| try: |
| counters_dongle_read_error.remove(name) |
| counters_dongle_connect_error.remove(name) |
| counters_dongle_rsrp_rsrq_error.remove(name) |
| counters_modem_cfun0_error.remove(name) |
| counters_modem_cfun1_error.remove(name) |
| counters_modem_cgatt_error.remove(name) |
| counters_modem_cesq_error.remove(name) |
| counters_dry_run_ping_error.remove(name) |
| counters_ping_error.remove(name) |
| counters_iperf_error.remove(name) |
| counters_report_send_error.remove(name) |
| except: |
| pass |
| |
| try: |
| maint_window.remove(name) |
| except: |
| pass |
| |
| @app.route('/edges/metrics', methods=['GET']) |
| def get_prometheus_metrics(): |
| res = [] |
| time_out_stale_results() |
| for edge in edges: |
| if edge['name'] == "ace-example": |
| continue |
| |
| connect_status = edge['status']['control_plane'] |
| ping_status = edge['status']['user_plane'] |
| |
| # Add PLMN Status |
| try: |
| if edge['dongle_stats']['PLMNStatus'] and edge['dongle_stats']['PLMNStatus'] in plmnstatus_mapping: |
| dongle_stats_plmnstatus.labels(edge['name']).set(plmnstatus_mapping[edge['dongle_stats']['PLMNStatus']]) |
| else: |
| pass |
| except: |
| pass |
| |
| # Add IMSI |
| try: |
| if edge['dongle_stats']['IMSI']: |
| dongle_stats_imsi.labels(edge['name']).set(float(edge['dongle_stats']['IMSI'])) |
| except: |
| pass |
| |
| # Add CellGlobalID |
| try: |
| if edge['dongle_stats']['CellGlobalID']: |
| dongle_stats_cellglobalid.labels(edge['name']).set(int(edge['dongle_stats']['CellGlobalID'], 16)) |
| except: |
| pass |
| |
| # Add PhyCellID |
| try: |
| if edge['dongle_stats']['PhyCellID']: |
| dongle_stats_phycellid.labels(edge['name']).set(edge['dongle_stats']['PhyCellID']) |
| except: |
| pass |
| |
| # Add ping dry_run latency results |
| ping_dry_run_min.labels(edge['name']).set(edge['speedtest']['ping']['dry_run']['min']) |
| ping_dry_run_avg.labels(edge['name']).set(edge['speedtest']['ping']['dry_run']['avg']) |
| ping_dry_run_max.labels(edge['name']).set(edge['speedtest']['ping']['dry_run']['max']) |
| ping_dry_run_stddev.labels(edge['name']).set(edge['speedtest']['ping']['dry_run']['stddev']) |
| ping_dry_run_transmitted.labels(edge['name']).set(edge['speedtest']['ping']['dry_run']['transmitted']) |
| ping_dry_run_received.labels(edge['name']).set(edge['speedtest']['ping']['dry_run']['received']) |
| ping_dry_run_median.labels(edge['name']).set(edge['speedtest']['ping']['dry_run']['median']) |
| |
| # Add ping dns latency results |
| ping_dns_min.labels(edge['name']).set(edge['speedtest']['ping']['dns']['min']) |
| ping_dns_avg.labels(edge['name']).set(edge['speedtest']['ping']['dns']['avg']) |
| ping_dns_max.labels(edge['name']).set(edge['speedtest']['ping']['dns']['max']) |
| ping_dns_stddev.labels(edge['name']).set(edge['speedtest']['ping']['dns']['stddev']) |
| ping_dns_transmitted.labels(edge['name']).set(edge['speedtest']['ping']['dns']['transmitted']) |
| ping_dns_received.labels(edge['name']).set(edge['speedtest']['ping']['dns']['received']) |
| ping_dns_median.labels(edge['name']).set(edge['speedtest']['ping']['dns']['median']) |
| |
| # Add ping iperf_server latency results |
| ping_iperf_server_min.labels(edge['name']).set(edge['speedtest']['ping']['iperf_server']['min']) |
| ping_iperf_server_avg.labels(edge['name']).set(edge['speedtest']['ping']['iperf_server']['avg']) |
| ping_iperf_server_max.labels(edge['name']).set(edge['speedtest']['ping']['iperf_server']['max']) |
| ping_iperf_server_stddev.labels(edge['name']).set(edge['speedtest']['ping']['iperf_server']['stddev']) |
| ping_iperf_server_transmitted.labels(edge['name']).set(edge['speedtest']['ping']['iperf_server']['transmitted']) |
| ping_iperf_server_received.labels(edge['name']).set(edge['speedtest']['ping']['iperf_server']['received']) |
| ping_iperf_server_median.labels(edge['name']).set(edge['speedtest']['ping']['iperf_server']['median']) |
| |
| # Add ping management_server latency results |
| ping_management_server_min.labels(edge['name']).set(edge['speedtest']['ping']['management_server']['min']) |
| ping_management_server_avg.labels(edge['name']).set(edge['speedtest']['ping']['management_server']['avg']) |
| ping_management_server_max.labels(edge['name']).set(edge['speedtest']['ping']['management_server']['max']) |
| ping_management_server_stddev.labels(edge['name']).set(edge['speedtest']['ping']['management_server']['stddev']) |
| ping_management_server_transmitted.labels(edge['name']).set(edge['speedtest']['ping']['management_server']['transmitted']) |
| ping_management_server_received.labels(edge['name']).set(edge['speedtest']['ping']['management_server']['received']) |
| ping_management_server_median.labels(edge['name']).set(edge['speedtest']['ping']['management_server']['median']) |
| |
| # Add iperf bandwidth results |
| iperf_cluster_downlink.labels(edge['name']).set(edge['speedtest']['iperf']['cluster']['downlink']) |
| iperf_cluster_uplink.labels(edge['name']).set(edge['speedtest']['iperf']['cluster']['uplink']) |
| |
| cp_status.labels(edge['name']).set(status_codes[connect_status]) |
| up_status.labels(edge['name']).set(status_codes[ping_status]) |
| |
| last_update.labels(edge['name']).set(edge['last_update']) |
| if 'maintenance' in edge: |
| maint_window.labels(edge['name']).set(int(edge['maintenance']['in_window'])) |
| |
| connect_test_ok.labels(edge['name']).set(0) |
| ping_test_ok.labels(edge['name']).set(0) |
| e2e_tests_ok.labels(edge['name']).set(0) |
| e2e_tests_down.labels(edge['name']).set(0) |
| |
| if connect_status in ["error", "no result"] or ping_status in ["error", "no result"]: |
| e2e_tests_down.labels(edge['name']).set(1) |
| else: |
| if connect_status == "connected": |
| connect_test_ok.labels(edge['name']).set(1) |
| if ping_status == "connected": |
| ping_test_ok.labels(edge['name']).set(1) |
| if connect_status == "connected" and ping_status == "connected": |
| e2e_tests_ok.labels(edge['name']).set(1) |
| |
| if 'signal_quality' in edge.keys(): |
| signal_quality_rsrq.labels(edge['name']).set(edge['signal_quality']['rsrq']) |
| signal_quality_rsrp.labels(edge['name']).set(edge['signal_quality']['rsrp']) |
| |
| if 'counters' in edge.keys(): |
| counters_dongle_read_error.labels(edge['name']).set(edge['counters']['dongle_read_error']) |
| counters_dongle_connect_error.labels(edge['name']).set(edge['counters']['dongle_connect_error']) |
| counters_dongle_rsrp_rsrq_error.labels(edge['name']).set(edge['counters']['dongle_rsrp_rsrq_error']) |
| counters_modem_cfun0_error.labels(edge['name']).set(edge['counters']['modem_cfun0_error']) |
| counters_modem_cfun1_error.labels(edge['name']).set(edge['counters']['modem_cfun1_error']) |
| counters_modem_cgatt_error.labels(edge['name']).set(edge['counters']['modem_cgatt_error']) |
| counters_modem_cesq_error.labels(edge['name']).set(edge['counters']['modem_cesq_error']) |
| counters_dry_run_ping_error.labels(edge['name']).set(edge['counters']['dry_run_ping_error']) |
| counters_ping_error.labels(edge['name']).set(edge['counters']['ping_error']) |
| counters_iperf_error.labels(edge['name']).set(edge['counters']['iperf_error']) |
| counters_report_send_error.labels(edge['name']).set(edge['counters']['report_send_error']) |
| |
| res.append(prom.generate_latest(cp_status)) |
| res.append(prom.generate_latest(up_status)) |
| |
| res.append(prom.generate_latest(dongle_stats_imsi)) |
| res.append(prom.generate_latest(dongle_stats_cellglobalid)) |
| res.append(prom.generate_latest(dongle_stats_plmnstatus)) |
| res.append(prom.generate_latest(dongle_stats_phycellid)) |
| |
| res.append(prom.generate_latest(ping_dry_run_min)) |
| res.append(prom.generate_latest(ping_dry_run_avg)) |
| res.append(prom.generate_latest(ping_dry_run_max)) |
| res.append(prom.generate_latest(ping_dry_run_stddev)) |
| res.append(prom.generate_latest(ping_dry_run_transmitted)) |
| res.append(prom.generate_latest(ping_dry_run_received)) |
| res.append(prom.generate_latest(ping_dry_run_median)) |
| |
| res.append(prom.generate_latest(ping_dns_min)) |
| res.append(prom.generate_latest(ping_dns_avg)) |
| res.append(prom.generate_latest(ping_dns_max)) |
| res.append(prom.generate_latest(ping_dns_stddev)) |
| res.append(prom.generate_latest(ping_dns_transmitted)) |
| res.append(prom.generate_latest(ping_dns_received)) |
| res.append(prom.generate_latest(ping_dns_median)) |
| |
| res.append(prom.generate_latest(ping_iperf_server_min)) |
| res.append(prom.generate_latest(ping_iperf_server_avg)) |
| res.append(prom.generate_latest(ping_iperf_server_max)) |
| res.append(prom.generate_latest(ping_iperf_server_stddev)) |
| res.append(prom.generate_latest(ping_iperf_server_transmitted)) |
| res.append(prom.generate_latest(ping_iperf_server_received)) |
| res.append(prom.generate_latest(ping_iperf_server_median)) |
| |
| res.append(prom.generate_latest(ping_management_server_min)) |
| res.append(prom.generate_latest(ping_management_server_avg)) |
| res.append(prom.generate_latest(ping_management_server_max)) |
| res.append(prom.generate_latest(ping_management_server_stddev)) |
| res.append(prom.generate_latest(ping_management_server_transmitted)) |
| res.append(prom.generate_latest(ping_management_server_received)) |
| res.append(prom.generate_latest(ping_management_server_median)) |
| |
| res.append(prom.generate_latest(iperf_cluster_downlink)) |
| res.append(prom.generate_latest(iperf_cluster_uplink)) |
| res.append(prom.generate_latest(last_update)) |
| res.append(prom.generate_latest(maint_window)) |
| res.append(prom.generate_latest(connect_test_ok)) |
| res.append(prom.generate_latest(ping_test_ok)) |
| res.append(prom.generate_latest(e2e_tests_ok)) |
| res.append(prom.generate_latest(e2e_tests_down)) |
| res.append(prom.generate_latest(signal_quality_rsrq)) |
| res.append(prom.generate_latest(signal_quality_rsrp)) |
| res.append(prom.generate_latest(counters_dongle_read_error)) |
| res.append(prom.generate_latest(counters_dongle_connect_error)) |
| res.append(prom.generate_latest(counters_dongle_rsrp_rsrq_error)) |
| res.append(prom.generate_latest(counters_modem_cfun0_error)) |
| res.append(prom.generate_latest(counters_modem_cfun1_error)) |
| res.append(prom.generate_latest(counters_modem_cgatt_error)) |
| res.append(prom.generate_latest(counters_modem_cesq_error)) |
| res.append(prom.generate_latest(counters_dry_run_ping_error)) |
| res.append(prom.generate_latest(counters_ping_error)) |
| res.append(prom.generate_latest(counters_iperf_error)) |
| res.append(prom.generate_latest(counters_report_send_error)) |
| |
| return Response(res, mimetype="text/plain") |
| |
| |
| @app.route('/edges/healthz', methods=['GET']) |
| def get_health(): |
| return {'message': 'healthy'} |
| |
| |
| @app.route('/edges', methods=['GET']) |
| def get_edges(): |
| time_out_stale_results() |
| return jsonify({'edges': edges}) |
| |
| |
| @app.route('/edges/<string:name>', methods=['GET']) |
| def get_edge(name): |
| time_out_stale_results() |
| edge = [edge for edge in edges if edge['name'] == name] |
| if len(edge) == 0: |
| abort(404) |
| return jsonify({'edge': edge[0]}) |
| |
| |
| @app.route('/edges', methods=['POST']) |
| @app.route('/testresults', methods=['POST']) |
| def create_or_update_edge(): |
| try: |
| jsonschema.validate(instance=request.json, schema=edgeSchema) |
| except jsonschema.exceptions.ValidationError as err: |
| app.logger.warn(err) |
| abort(400) |
| |
| req_edge = { |
| 'name': request.json['name'], |
| 'status': { |
| 'control_plane': request.json['status']['control_plane'], |
| 'user_plane': request.json['status']['user_plane'] |
| }, |
| 'dongle_stats': { |
| 'SuccessfulFetch': False |
| }, |
| 'speedtest': { |
| 'ping': { |
| 'dry_run': { |
| 'transmitted': 0, |
| 'received': 0, |
| 'median': 0.0, |
| 'min': 0.0, |
| 'avg': 0.0, |
| 'max': 0.0, |
| 'stddev': 0.0 |
| }, |
| 'dns': { |
| 'transmitted' : 0, |
| 'received' : 0, |
| 'median' : 0.0, |
| 'min': 0.0, |
| 'avg': 0.0, |
| 'max': 0.0, |
| 'stddev': 0.0 |
| }, |
| 'iperf_server': { |
| 'transmitted' : 0, |
| 'received' : 0, |
| 'median' : 0.0, |
| 'min': 0.0, |
| 'avg': 0.0, |
| 'max': 0.0, |
| 'stddev': 0.0 |
| }, |
| 'management_server': { |
| 'transmitted' : 0, |
| 'received' : 0, |
| 'median' : 0.0, |
| 'min': 0.0, |
| 'avg': 0.0, |
| 'max': 0.0, |
| 'stddev': 0.0 |
| } |
| }, |
| 'iperf': { |
| 'cluster': { |
| 'downlink': 0.0, |
| 'uplink': 0.0 |
| } |
| } |
| }, |
| 'counters': { |
| 'dongle_read_error': 0, |
| 'dongle_connect_error': 0, |
| 'dongle_rsrp_rsrq_error': 0, |
| 'modem_cfun0_error': 0, |
| 'modem_cfun1_error': 0, |
| 'modem_cgatt_error': 0, |
| 'modem_cesq_error': 0, |
| 'dry_run_ping_error': 0, |
| 'ping_error': 0, |
| 'iperf_error': 0, |
| 'report_send_error': 0 |
| }, |
| 'last_update': time.time() |
| } |
| |
| if 'speedtest' in request.json: |
| if 'ping' in request.json['speedtest']: |
| req_edge['speedtest']['ping'] = request.json['speedtest']['ping'] |
| if 'iperf' in request.json['speedtest']: |
| req_edge['speedtest']['iperf'] = request.json['speedtest']['iperf'] |
| |
| if 'signal_quality' in request.json: |
| req_edge['signal_quality'] = request.json['signal_quality'] |
| |
| if 'dongle_stats' in request.json: |
| req_edge['dongle_stats'] = request.json['dongle_stats'] |
| |
| if 'counters' in request.json: |
| req_edge['counters'] = request.json['counters'] |
| |
| edge = [edge for edge in edges if edge['name'] == req_edge['name']] |
| if len(edge) == 0: |
| app.logger.info("new edge request " + req_edge['name']) |
| edges.append(req_edge) |
| else: |
| edge[0]['status']['control_plane'] = req_edge['status']['control_plane'] |
| edge[0]['status']['user_plane'] = req_edge['status']['user_plane'] |
| edge[0]['speedtest']['ping'] = req_edge['speedtest']['ping'] |
| edge[0]['speedtest']['iperf'] = req_edge['speedtest']['iperf'] |
| edge[0]['dongle_stats'] = req_edge['dongle_stats'] |
| if 'signal_quality' in req_edge.keys(): |
| edge[0]['signal_quality'] = req_edge['signal_quality'] |
| if 'counters' in req_edge.keys(): |
| edge[0]['counters'] = req_edge['counters'] |
| edge[0]['last_update'] = req_edge['last_update'] |
| |
| return jsonify({'edge': req_edge}), 201 |
| |
| |
| @app.route('/edges/<string:name>', methods=['DELETE']) |
| @app.route('/testresults/<string:name>', methods=['DELETE']) |
| def delete_edge(name): |
| app.logger.info("delete edge request " + name) |
| result = False |
| for i in range(len(edges)): |
| if edges[i]['name'] == name: |
| del edges[i] |
| remove_edge_from_metrics(name) |
| result = True |
| break |
| if not result: |
| abort(404) |
| return jsonify({'result': True}) |
| |
| |
| if __name__ == '__main__': |
| if SECRET_ICAL_URL and AETHER_ENV: |
| app.logger.info(" * Starting maintenance calendar polling thread (Aether env: %s)" % AETHER_ENV) |
| t = threading.Thread(target=pull_maintenance_events) |
| t.start() |
| app.run(debug=True, host='0.0.0.0', port=80) |