[AETHER-2613]: Improve Ping Test of Monitoring Agent
Ping entries added (taken from ping command output):
- transmitted: The number of packets transmitted.
- received: The number of packets received.
- median: The median of all successful rtt packet times in ms.
Adds support for running ping to+from:
- management_server
- dry_run: Should be first to run since first ping test after attach may drop packets. Runs 3 iterations.
Also changes default DNS to 1.1.1.1
Change-Id: I796ce1bd88510282703e9400a69cf5df31f9fc53
diff --git a/edge-monitoring-server/edge_monitoring_server.py b/edge-monitoring-server/edge_monitoring_server.py
index f9184d8..0110e34 100755
--- a/edge-monitoring-server/edge_monitoring_server.py
+++ b/edge-monitoring-server/edge_monitoring_server.py
@@ -61,9 +61,25 @@
"ping": {
"type": "object",
"properties": {
+ "dry_run": {
+ "type": "object",
+ "properties": {
+ "transmitted": {"type": "number"},
+ "received": {"type": "number"},
+ "median": {"type": "number"},
+ "min": {"type": "number"},
+ "avg": {"type": "number"},
+ "max": {"type": "number"},
+ "stddev": {"type": "number"}
+ },
+ "required": ["min", "avg", "max", "stddev"]
+ },
"dns": {
"type": "object",
"properties": {
+ "transmitted": {"type": "number"},
+ "received": {"type": "number"},
+ "median": {"type": "number"},
"min": {"type": "number"},
"avg": {"type": "number"},
"max": {"type": "number"},
@@ -74,13 +90,29 @@
"iperf_server": {
"type": "object",
"properties": {
+ "transmitted": {"type": "number"},
+ "received": {"type": "number"},
+ "median": {"type": "number"},
"min": {"type": "number"},
"avg": {"type": "number"},
"max": {"type": "number"},
"stddev": {"type": "number"}
},
"required": ["min", "avg", "max", "stddev"]
- }
+ },
+ "management_server": {
+ "type": "object",
+ "properties": {
+ "transmitted": {"type": "number"},
+ "received": {"type": "number"},
+ "median": {"type": "number"},
+ "min": {"type": "number"},
+ "avg": {"type": "number"},
+ "max": {"type": "number"},
+ "stddev": {"type": "number"}
+ },
+ "required": ["min", "avg", "max", "stddev"]
+ },
}
},
"iperf": {
@@ -119,13 +151,37 @@
},
'speedtest': {
'ping': {
+ 'dry_run': {
+ 'transmitted': 0,
+ 'received': 0,
+ 'median': 0.0,
+ 'min': 0.0,
+ 'avg': 0.0,
+ 'max': 0.0,
+ 'stddev': 0.0
+ },
'dns': {
+ 'transmitted': 0,
+ 'received': 0,
+ 'median': 0.0,
'min': 0.0,
'avg': 0.0,
'max': 0.0,
'stddev': 0.0
},
'iperf_server': {
+ 'transmitted': 0,
+ 'received': 0,
+ 'median': 0.0,
+ 'min': 0.0,
+ 'avg': 0.0,
+ 'max': 0.0,
+ 'stddev': 0.0
+ },
+ 'management_server': {
+ 'transmitted': 0,
+ 'received': 0,
+ 'median': 0.0,
'min': 0.0,
'avg': 0.0,
'max': 0.0,
@@ -170,18 +226,42 @@
ping_test_ok = prom.Gauge("aetheredge_ping_test_ok", "Last ping test passed", ["name"])
e2e_tests_down = prom.Gauge("aetheredge_e2e_tests_down", "E2E tests not reporting", ["name"])
-# Speedtest dns ping metrics
+# Ping dry_run metrics
+ping_dry_run_transmitted = prom.Gauge("aetheredge_ping_dry_run_test_transmitted","Last ping test to dry_run number of transmitted packets",["name"])
+ping_dry_run_received = prom.Gauge("aetheredge_ping_dry_run_test_received","Last ping test to dry_run number of received packets",["name"])
+ping_dry_run_median = prom.Gauge("aetheredge_ping_dry_run_test_median","Last ping test to dry_run median value",["name"])
+ping_dry_run_min = prom.Gauge("aetheredge_ping_dry_run_test_min","Last ping test to dry_run minimum value",["name"])
+ping_dry_run_avg = prom.Gauge("aetheredge_ping_dry_run_test_avg","Last ping test to dry_run average",["name"])
+ping_dry_run_max = prom.Gauge("aetheredge_ping_dry_run_test_max","Last ping test to dry_run maximum value",["name"])
+ping_dry_run_stddev = prom.Gauge("aetheredge_ping_dry_run_test_stddev","Last ping test to dry_run standard deviation",["name"])
+
+# Ping dns metrics
+ping_dns_transmitted = prom.Gauge("aetheredge_ping_dns_test_transmitted","Last ping test to dns number of transmitted packets",["name"])
+ping_dns_received = prom.Gauge("aetheredge_ping_dns_test_received","Last ping test to dns number of received packets",["name"])
+ping_dns_median = prom.Gauge("aetheredge_ping_dns_test_median","Last ping test to dns median value",["name"])
ping_dns_min = prom.Gauge("aetheredge_ping_dns_test_min","Last ping test to dns minimum value",["name"])
ping_dns_avg = prom.Gauge("aetheredge_ping_dns_test_avg","Last ping test to dns average",["name"])
ping_dns_max = prom.Gauge("aetheredge_ping_dns_test_max","Last ping test to dns maximum value",["name"])
ping_dns_stddev = prom.Gauge("aetheredge_ping_dns_test_stddev","Last ping test to dns standard deviation",["name"])
-# Speedtest iperf server ping metrics
+# Ping iperf server metrics
+ping_iperf_server_transmitted = prom.Gauge("aetheredge_ping_iperf_server_test_transmitted","Last ping test to iperf_server number of transmitted packets",["name"])
+ping_iperf_server_received = prom.Gauge("aetheredge_ping_iperf_server_test_received","Last ping test to iperf_server number of received packets",["name"])
+ping_iperf_server_median = prom.Gauge("aetheredge_ping_iperf_server_test_median","Last ping test to iperf_server median value",["name"])
ping_iperf_server_min = prom.Gauge("aetheredge_ping_iperf_server_test_min","Last ping test to iperf_server minimum value",["name"])
ping_iperf_server_avg = prom.Gauge("aetheredge_ping_iperf_server_test_avg","Last ping test to iperf_server average",["name"])
ping_iperf_server_max = prom.Gauge("aetheredge_ping_iperf_server_test_max","Last ping test to iperf_server maximum value",["name"])
ping_iperf_server_stddev = prom.Gauge("aetheredge_ping_iperf_server_test_stddev","Last ping test to iperf_server standard deviation",["name"])
+# Ping management_server metrics
+ping_management_server_transmitted = prom.Gauge("aetheredge_ping_management_server_test_transmitted","Last ping test to management_server number of transmitted packets",["name"])
+ping_management_server_received = prom.Gauge("aetheredge_ping_management_server_test_received","Last ping test to management_server number of received packets",["name"])
+ping_management_server_median = prom.Gauge("aetheredge_ping_management_server_test_median","Last ping test to management_server median value",["name"])
+ping_management_server_min = prom.Gauge("aetheredge_ping_management_server_test_min","Last ping test to management_server minimum value",["name"])
+ping_management_server_avg = prom.Gauge("aetheredge_ping_management_server_test_avg","Last ping test to management_server average",["name"])
+ping_management_server_max = prom.Gauge("aetheredge_ping_management_server_test_max","Last ping test to management_server maximum value",["name"])
+ping_management_server_stddev = prom.Gauge("aetheredge_ping_management_server_test_stddev","Last ping test to management_server standard deviation",["name"])
+
# Speedtest iperf metrics
iperf_cluster_downlink = prom.Gauge("aetheredge_iperf_cluster_downlink_test","Last iperf test downlink result",["name"])
iperf_cluster_uplink = prom.Gauge("aetheredge_iperf_cluster_uplink_test","Last iperf test downlink result",["name"])
@@ -249,14 +329,34 @@
if time_elapsed > NO_RESULT_THRESHOLD:
edge['status']['control_plane'] = "no result"
edge['status']['user_plane'] = "no result"
- edge['speedtest']['ping']['dns'] = {'min': 0.0,
+ edge['speedtest']['ping']['dry_run'] = {'transmitted': 0,
+ 'received': 0,
+ 'median': 0.0,
+ 'min': 0.0,
+ 'avg': 0.0,
+ 'max': 0.0,
+ 'stddev': 0.0}
+ edge['speedtest']['ping']['dns'] = {'transmitted': 0,
+ 'received': 0,
+ 'median': 0.0,
+ 'min': 0.0,
'avg': 0.0,
'max': 0.0,
'stddev': 0.0}
- edge['speedtest']['ping']['iperf_server'] = {'min': 0.0,
+ edge['speedtest']['ping']['iperf_server'] = {'transmitted': 0,
+ 'received': 0,
+ 'median': 0.0,
+ 'min': 0.0,
'avg': 0.0,
'max': 0.0,
'stddev': 0.0}
+ edge['speedtest']['ping']['management_server'] = {'transmitted': 0,
+ 'received': 0,
+ 'median': 0.0,
+ 'min': 0.0,
+ 'avg': 0.0,
+ 'max': 0.0,
+ 'stddev': 0.0}
edge['speedtest']['iperf'] = {'cluster': {
'downlink': 0.0,
'uplink': 0.0
@@ -277,10 +377,24 @@
pass
try:
+ ping_dry_run_min.remove(name)
+ ping_dry_run_avg.remove(name)
+ ping_dry_run_max.remove(name)
+ ping_dry_run_stddev.remove(name)
+ ping_dry_run_transmitted.remove(name)
+ ping_dry_run_received.remove(name)
+ ping_dry_run_median.remove(name)
+ except:
+ pass
+
+ try:
ping_dns_min.remove(name)
ping_dns_avg.remove(name)
ping_dns_max.remove(name)
ping_dns_stddev.remove(name)
+ ping_dns_transmitted.remove(name)
+ ping_dns_received.remove(name)
+ ping_dns_median.remove(name)
except:
pass
@@ -289,6 +403,20 @@
ping_iperf_server_avg.remove(name)
ping_iperf_server_max.remove(name)
ping_iperf_server_stddev.remove(name)
+ ping_iperf_server_transmitted.remove(name)
+ ping_iperf_server_received.remove(name)
+ ping_iperf_server_median.remove(name)
+ except:
+ pass
+
+ try:
+ ping_management_server_min.remove(name)
+ ping_management_server_avg.remove(name)
+ ping_management_server_max.remove(name)
+ ping_management_server_stddev.remove(name)
+ ping_management_server_transmitted.remove(name)
+ ping_management_server_received.remove(name)
+ ping_management_server_median.remove(name)
except:
pass
@@ -320,6 +448,19 @@
connect_status = edge['status']['control_plane']
ping_status = edge['status']['user_plane']
+ # Add ping dry_run latency results if available
+ try:
+ if edge['speedtest']['ping']['dry_run']['avg']:
+ ping_dry_run_min.labels(edge['name']).set(edge['speedtest']['ping']['dry_run']['min'])
+ ping_dry_run_avg.labels(edge['name']).set(edge['speedtest']['ping']['dry_run']['avg'])
+ ping_dry_run_max.labels(edge['name']).set(edge['speedtest']['ping']['dry_run']['max'])
+ ping_dry_run_stddev.labels(edge['name']).set(edge['speedtest']['ping']['dry_run']['stddev'])
+ ping_dry_run_transmitted.labels(edge['name']).set(edge['speedtest']['ping']['dry_run']['transmitted'])
+ ping_dry_run_received.labels(edge['name']).set(edge['speedtest']['ping']['dry_run']['received'])
+ ping_dry_run_median.labels(edge['name']).set(edge['speedtest']['ping']['dry_run']['median'])
+ except KeyError:
+ pass
+
# Add ping dns latency results if available
try:
if edge['speedtest']['ping']['dns']['avg']:
@@ -327,6 +468,9 @@
ping_dns_avg.labels(edge['name']).set(edge['speedtest']['ping']['dns']['avg'])
ping_dns_max.labels(edge['name']).set(edge['speedtest']['ping']['dns']['max'])
ping_dns_stddev.labels(edge['name']).set(edge['speedtest']['ping']['dns']['stddev'])
+ ping_dns_transmitted.labels(edge['name']).set(edge['speedtest']['ping']['dns']['transmitted'])
+ ping_dns_received.labels(edge['name']).set(edge['speedtest']['ping']['dns']['received'])
+ ping_dns_median.labels(edge['name']).set(edge['speedtest']['ping']['dns']['median'])
except KeyError:
pass
@@ -337,6 +481,22 @@
ping_iperf_server_avg.labels(edge['name']).set(edge['speedtest']['ping']['iperf_server']['avg'])
ping_iperf_server_max.labels(edge['name']).set(edge['speedtest']['ping']['iperf_server']['max'])
ping_iperf_server_stddev.labels(edge['name']).set(edge['speedtest']['ping']['iperf_server']['stddev'])
+ ping_iperf_server_transmitted.labels(edge['name']).set(edge['speedtest']['ping']['iperf_server']['transmitted'])
+ ping_iperf_server_received.labels(edge['name']).set(edge['speedtest']['ping']['iperf_server']['received'])
+ ping_iperf_server_median.labels(edge['name']).set(edge['speedtest']['ping']['iperf_server']['median'])
+ except KeyError:
+ pass
+
+ # Add ping management_server latency results if available
+ try:
+ if edge['speedtest']['ping']['management_server']['avg']:
+ ping_management_server_min.labels(edge['name']).set(edge['speedtest']['ping']['management_server']['min'])
+ ping_management_server_avg.labels(edge['name']).set(edge['speedtest']['ping']['management_server']['avg'])
+ ping_management_server_max.labels(edge['name']).set(edge['speedtest']['ping']['management_server']['max'])
+ ping_management_server_stddev.labels(edge['name']).set(edge['speedtest']['ping']['management_server']['stddev'])
+ ping_management_server_transmitted.labels(edge['name']).set(edge['speedtest']['ping']['management_server']['transmitted'])
+ ping_management_server_received.labels(edge['name']).set(edge['speedtest']['ping']['management_server']['received'])
+ ping_management_server_median.labels(edge['name']).set(edge['speedtest']['ping']['management_server']['median'])
except KeyError:
pass
@@ -376,14 +536,39 @@
res.append(prom.generate_latest(cp_status))
res.append(prom.generate_latest(up_status))
+
+ res.append(prom.generate_latest(ping_dry_run_min))
+ res.append(prom.generate_latest(ping_dry_run_avg))
+ res.append(prom.generate_latest(ping_dry_run_max))
+ res.append(prom.generate_latest(ping_dry_run_stddev))
+ res.append(prom.generate_latest(ping_dry_run_transmitted))
+ res.append(prom.generate_latest(ping_dry_run_received))
+ res.append(prom.generate_latest(ping_dry_run_median))
+
res.append(prom.generate_latest(ping_dns_min))
res.append(prom.generate_latest(ping_dns_avg))
res.append(prom.generate_latest(ping_dns_max))
res.append(prom.generate_latest(ping_dns_stddev))
+ res.append(prom.generate_latest(ping_dns_transmitted))
+ res.append(prom.generate_latest(ping_dns_received))
+ res.append(prom.generate_latest(ping_dns_median))
+
res.append(prom.generate_latest(ping_iperf_server_min))
res.append(prom.generate_latest(ping_iperf_server_avg))
res.append(prom.generate_latest(ping_iperf_server_max))
res.append(prom.generate_latest(ping_iperf_server_stddev))
+ res.append(prom.generate_latest(ping_iperf_server_transmitted))
+ res.append(prom.generate_latest(ping_iperf_server_received))
+ res.append(prom.generate_latest(ping_iperf_server_median))
+
+ res.append(prom.generate_latest(ping_management_server_min))
+ res.append(prom.generate_latest(ping_management_server_avg))
+ res.append(prom.generate_latest(ping_management_server_max))
+ res.append(prom.generate_latest(ping_management_server_stddev))
+ res.append(prom.generate_latest(ping_management_server_transmitted))
+ res.append(prom.generate_latest(ping_management_server_received))
+ res.append(prom.generate_latest(ping_management_server_median))
+
res.append(prom.generate_latest(iperf_cluster_downlink))
res.append(prom.generate_latest(iperf_cluster_uplink))
res.append(prom.generate_latest(last_update))
@@ -435,13 +620,37 @@
},
'speedtest': {
'ping': {
+ 'dry_run': {
+ 'transmitted' : 0,
+ 'received' : 0,
+ 'median' : 0.0,
+ 'min': 0.0,
+ 'avg': 0.0,
+ 'max': 0.0,
+ 'stddev': 0.0
+ },
'dns': {
+ 'transmitted' : 0,
+ 'received' : 0,
+ 'median' : 0.0,
'min': 0.0,
'avg': 0.0,
'max': 0.0,
'stddev': 0.0
},
'iperf_server': {
+ 'transmitted' : 0,
+ 'received' : 0,
+ 'median' : 0.0,
+ 'min': 0.0,
+ 'avg': 0.0,
+ 'max': 0.0,
+ 'stddev': 0.0
+ },
+ 'management_server': {
+ 'transmitted' : 0,
+ 'received' : 0,
+ 'median' : 0.0,
'min': 0.0,
'avg': 0.0,
'max': 0.0,