Gitiles
Code Review Sign In
gerrit.opencord.org / helm-charts / 40a144bd42d8cf7672a767a8334c14ce86797f2e / . / nem-monitoring / grafana-dashboards / etcd-metrics.json
blob: cc7d4b8b41c2a0f1cc996f6965c832bf0bd48b75 [file] [log] [blame]
Shrey Baidb311fee2020-05-22 16:48:00 +0530[diff] [blame]1{
2 "annotations": {
3 "list": [
4 {
5 "builtIn": 1,
6 "datasource": "-- Grafana --",
7 "enable": true,
8 "hide": true,
9 "iconColor": "rgba(0, 211, 255, 1)",
10 "name": "Annotations & Alerts",
11 "type": "dashboard"
12 }
13 ]
14 },
15 "description": "Etcd Dashboard for Prometheus metrics scraper",
16 "editable": true,
17 "gnetId": 3070,
18 "graphTooltip": 0,
19 "id": 9,
20 "links": [],
21 "panels": [
22 {
23 "cacheTimeout": null,
24 "colorBackground": false,
25 "colorValue": true,
26 "colors": [
27 "rgba(245, 54, 54, 0.9)",
28 "rgba(237, 129, 40, 0.89)",
29 "rgba(50, 172, 45, 0.97)"
30 ],
31 "datasource": "Prometheus",
32 "format": "none",
33 "gauge": {
34 "maxValue": 100,
35 "minValue": 0,
36 "show": false,
37 "thresholdLabels": false,
38 "thresholdMarkers": true
39 },
40 "gridPos": {
41 "h": 7,
42 "w": 8,
43 "x": 0,
44 "y": 0
45 },
46 "id": 44,
47 "interval": null,
48 "links": [],
49 "mappingType": 1,
50 "mappingTypes": [
51 {
52 "name": "value to text",
53 "value": 1
54 },
55 {
56 "name": "range to text",
57 "value": 2
58 }
59 ],
60 "maxDataPoints": 100,
61 "nullPointMode": "connected",
62 "nullText": null,
63 "options": {},
64 "postfix": "",
65 "postfixFontSize": "50%",
66 "prefix": "",
67 "prefixFontSize": "50%",
68 "rangeMaps": [
69 {
70 "from": "null",
71 "text": "N/A",
72 "to": "null"
73 }
74 ],
75 "sparkline": {
76 "fillColor": "rgba(31, 118, 189, 0.18)",
77 "full": false,
78 "lineColor": "rgb(31, 120, 193)",
79 "show": false
80 },
81 "tableColumn": "",
82 "targets": [
83 {
84 "expr": "max(etcd_server_has_leader)",
85 "format": "time_series",
86 "intervalFactor": 2,
87 "refId": "A",
88 "step": 600
89 }
90 ],
91 "thresholds": "0,1",
92 "title": "Etcd has a leader?",
93 "type": "singlestat",
94 "valueFontSize": "80%",
95 "valueMaps": [
96 {
97 "op": "=",
98 "text": "YES",
99 "value": "1"
100 },
101 {
102 "op": "=",
103 "text": "NO",
104 "value": "0"
105 }
106 ],
107 "valueName": "avg"
108 },
109 {
110 "cacheTimeout": null,
111 "colorBackground": false,
112 "colorValue": false,
113 "colors": [
114 "rgba(245, 54, 54, 0.9)",
115 "rgba(237, 129, 40, 0.89)",
116 "rgba(50, 172, 45, 0.97)"
117 ],
118 "datasource": "Prometheus",
119 "format": "none",
120 "gauge": {
121 "maxValue": 100,
122 "minValue": 0,
123 "show": false,
124 "thresholdLabels": false,
125 "thresholdMarkers": true
126 },
127 "gridPos": {
128 "h": 7,
129 "w": 8,
130 "x": 8,
131 "y": 0
132 },
133 "id": 42,
134 "interval": null,
135 "links": [],
136 "mappingType": 1,
137 "mappingTypes": [
138 {
139 "name": "value to text",
140 "value": 1
141 },
142 {
143 "name": "range to text",
144 "value": 2
145 }
146 ],
147 "maxDataPoints": 100,
148 "nullPointMode": "connected",
149 "nullText": null,
150 "options": {},
151 "postfix": "",
152 "postfixFontSize": "50%",
153 "prefix": "",
154 "prefixFontSize": "50%",
155 "rangeMaps": [
156 {
157 "from": "null",
158 "text": "N/A",
159 "to": "null"
160 }
161 ],
162 "sparkline": {
163 "fillColor": "rgba(31, 118, 189, 0.18)",
164 "full": false,
165 "lineColor": "rgb(31, 120, 193)",
166 "show": false
167 },
168 "tableColumn": "",
169 "targets": [
170 {
171 "expr": "max(etcd_server_leader_changes_seen_total)",
172 "format": "time_series",
173 "intervalFactor": 2,
174 "refId": "A",
175 "step": 600
176 }
177 ],
178 "thresholds": "",
179 "title": "The number of leader changes seen",
180 "type": "singlestat",
181 "valueFontSize": "80%",
182 "valueMaps": [
183 {
184 "op": "=",
185 "text": "N/A",
186 "value": "null"
187 }
188 ],
189 "valueName": "avg"
190 },
191 {
192 "cacheTimeout": null,
193 "colorBackground": false,
194 "colorValue": false,
195 "colors": [
196 "rgba(245, 54, 54, 0.9)",
197 "rgba(237, 129, 40, 0.89)",
198 "rgba(50, 172, 45, 0.97)"
199 ],
200 "datasource": "Prometheus",
201 "format": "none",
202 "gauge": {
203 "maxValue": 100,
204 "minValue": 0,
205 "show": false,
206 "thresholdLabels": false,
207 "thresholdMarkers": true
208 },
209 "gridPos": {
210 "h": 7,
211 "w": 8,
212 "x": 16,
213 "y": 0
214 },
215 "id": 43,
216 "interval": null,
217 "links": [],
218 "mappingType": 1,
219 "mappingTypes": [
220 {
221 "name": "value to text",
222 "value": 1
223 },
224 {
225 "name": "range to text",
226 "value": 2
227 }
228 ],
229 "maxDataPoints": 100,
230 "nullPointMode": "connected",
231 "nullText": null,
232 "options": {},
233 "postfix": "",
234 "postfixFontSize": "50%",
235 "prefix": "",
236 "prefixFontSize": "50%",
237 "rangeMaps": [
238 {
239 "from": "null",
240 "text": "N/A",
241 "to": "null"
242 }
243 ],
244 "sparkline": {
245 "fillColor": "rgba(31, 118, 189, 0.18)",
246 "full": false,
247 "lineColor": "rgb(31, 120, 193)",
248 "show": false
249 },
250 "tableColumn": "",
251 "targets": [
252 {
253 "expr": "max(etcd_server_leader_changes_seen_total)",
254 "format": "time_series",
255 "intervalFactor": 2,
256 "refId": "A",
257 "step": 600
258 }
259 ],
260 "thresholds": "",
261 "title": "The total number of failed proposals seen",
262 "type": "singlestat",
263 "valueFontSize": "80%",
264 "valueMaps": [
265 {
266 "op": "=",
267 "text": "N/A",
268 "value": "null"
269 }
270 ],
271 "valueName": "avg"
272 },
273 {
274 "aliasColors": {},
275 "bars": false,
276 "dashLength": 10,
277 "dashes": false,
278 "datasource": "Prometheus",
279 "editable": true,
280 "error": false,
281 "fill": 0,
282 "fillGradient": 0,
283 "gridPos": {
284 "h": 7,
285 "w": 12,
286 "x": 0,
287 "y": 7
288 },
289 "id": 23,
290 "legend": {
291 "avg": false,
292 "current": false,
293 "max": false,
294 "min": false,
295 "show": false,
296 "total": false,
297 "values": false
298 },
299 "lines": true,
300 "linewidth": 2,
301 "links": [],
302 "nullPointMode": "connected",
303 "options": {
304 "dataLinks": []
305 },
306 "percentage": false,
307 "pointradius": 5,
308 "points": false,
309 "renderer": "flot",
310 "seriesOverrides": [],
311 "spaceLength": 10,
312 "stack": false,
313 "steppedLine": false,
314 "targets": [
315 {
316 "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[5m]))",
317 "format": "time_series",
318 "intervalFactor": 2,
319 "legendFormat": "RPC Rate",
320 "metric": "grpc_server_started_total",
321 "refId": "A",
322 "step": 60
323 },
324 {
325 "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[5m]))",
326 "format": "time_series",
327 "intervalFactor": 2,
328 "legendFormat": "RPC Failed Rate",
329 "metric": "grpc_server_handled_total",
330 "refId": "B",
331 "step": 60
332 }
333 ],
334 "thresholds": [],
335 "timeFrom": null,
336 "timeRegions": [],
337 "timeShift": null,
338 "title": "RPC Rate",
339 "tooltip": {
340 "msResolution": false,
341 "shared": true,
342 "sort": 0,
343 "value_type": "individual"
344 },
345 "type": "graph",
346 "xaxis": {
347 "buckets": null,
348 "mode": "time",
349 "name": null,
350 "show": true,
351 "values": []
352 },
353 "yaxes": [
354 {
355 "format": "ops",
356 "label": null,
357 "logBase": 1,
358 "max": null,
359 "min": null,
360 "show": true
361 },
362 {
363 "format": "short",
364 "label": null,
365 "logBase": 1,
366 "max": null,
367 "min": null,
368 "show": true
369 }
370 ],
371 "yaxis": {
372 "align": false,
373 "alignLevel": null
374 }
375 },
376 {
377 "aliasColors": {},
378 "bars": false,
379 "dashLength": 10,
380 "dashes": false,
381 "datasource": "Prometheus",
382 "editable": true,
383 "error": false,
384 "fill": 0,
385 "fillGradient": 0,
386 "gridPos": {
387 "h": 7,
388 "w": 12,
389 "x": 12,
390 "y": 7
391 },
392 "id": 41,
393 "legend": {
394 "avg": false,
395 "current": false,
396 "max": false,
397 "min": false,
398 "show": false,
399 "total": false,
400 "values": false
401 },
402 "lines": true,
403 "linewidth": 2,
404 "links": [],
405 "nullPointMode": "connected",
406 "options": {
407 "dataLinks": []
408 },
409 "percentage": false,
410 "pointradius": 5,
411 "points": false,
412 "renderer": "flot",
413 "seriesOverrides": [],
414 "spaceLength": 10,
415 "stack": true,
416 "steppedLine": false,
417 "targets": [
418 {
419 "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})",
420 "format": "time_series",
421 "intervalFactor": 2,
422 "legendFormat": "Watch Streams",
423 "metric": "grpc_server_handled_total",
424 "refId": "A",
425 "step": 60
426 },
427 {
428 "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})",
429 "format": "time_series",
430 "intervalFactor": 2,
431 "legendFormat": "Lease Streams",
432 "metric": "grpc_server_handled_total",
433 "refId": "B",
434 "step": 60
435 }
436 ],
437 "thresholds": [],
438 "timeFrom": null,
439 "timeRegions": [],
440 "timeShift": null,
441 "title": "Active Streams",
442 "tooltip": {
443 "msResolution": false,
444 "shared": true,
445 "sort": 0,
446 "value_type": "individual"
447 },
448 "type": "graph",
449 "xaxis": {
450 "buckets": null,
451 "mode": "time",
452 "name": null,
453 "show": true,
454 "values": []
455 },
456 "yaxes": [
457 {
458 "format": "short",
459 "label": "",
460 "logBase": 1,
461 "max": null,
462 "min": null,
463 "show": true
464 },
465 {
466 "format": "short",
467 "label": null,
468 "logBase": 1,
469 "max": null,
470 "min": null,
471 "show": true
472 }
473 ],
474 "yaxis": {
475 "align": false,
476 "alignLevel": null
477 }
478 },
479 {
480 "aliasColors": {},
481 "bars": false,
482 "dashLength": 10,
483 "dashes": false,
484 "datasource": "Prometheus",
485 "decimals": null,
486 "editable": true,
487 "error": false,
488 "fill": 0,
489 "fillGradient": 0,
490 "grid": {},
491 "gridPos": {
492 "h": 7,
493 "w": 8,
494 "x": 0,
495 "y": 14
496 },
497 "id": 1,
498 "legend": {
499 "avg": false,
500 "current": false,
501 "max": false,
502 "min": false,
503 "show": false,
504 "total": false,
505 "values": false
506 },
507 "lines": true,
508 "linewidth": 2,
509 "links": [],
510 "nullPointMode": "connected",
511 "options": {
512 "dataLinks": []
513 },
514 "percentage": false,
515 "pointradius": 5,
516 "points": false,
517 "renderer": "flot",
518 "seriesOverrides": [],
519 "spaceLength": 10,
520 "stack": false,
521 "steppedLine": false,
522 "targets": [
523 {
524 "expr": "etcd_debugging_mvcc_db_total_size_in_bytes",
525 "format": "time_series",
526 "hide": false,
527 "interval": "",
528 "intervalFactor": 2,
529 "legendFormat": "{{instance}} DB Size",
530 "metric": "",
531 "refId": "A",
532 "step": 120
533 }
534 ],
535 "thresholds": [],
536 "timeFrom": null,
537 "timeRegions": [],
538 "timeShift": null,
539 "title": "DB Size",
540 "tooltip": {
541 "msResolution": false,
542 "shared": true,
543 "sort": 0,
544 "value_type": "cumulative"
545 },
546 "type": "graph",
547 "xaxis": {
548 "buckets": null,
549 "mode": "time",
550 "name": null,
551 "show": true,
552 "values": []
553 },
554 "yaxes": [
555 {
556 "format": "bytes",
557 "logBase": 1,
558 "max": null,
559 "min": null,
560 "show": true
561 },
562 {
563 "format": "short",
564 "logBase": 1,
565 "max": null,
566 "min": null,
567 "show": false
568 }
569 ],
570 "yaxis": {
571 "align": false,
572 "alignLevel": null
573 }
574 },
575 {
576 "aliasColors": {},
577 "bars": false,
578 "dashLength": 10,
579 "dashes": false,
580 "datasource": "Prometheus",
581 "editable": true,
582 "error": false,
583 "fill": 0,
584 "fillGradient": 0,
585 "grid": {},
586 "gridPos": {
587 "h": 7,
588 "w": 8,
589 "x": 8,
590 "y": 14
591 },
592 "id": 3,
593 "legend": {
594 "avg": false,
595 "current": false,
596 "max": false,
597 "min": false,
598 "show": false,
599 "total": false,
600 "values": false
601 },
602 "lines": true,
603 "linewidth": 2,
604 "links": [],
605 "nullPointMode": "connected",
606 "options": {
607 "dataLinks": []
608 },
609 "percentage": false,
610 "pointradius": 1,
611 "points": false,
612 "renderer": "flot",
613 "seriesOverrides": [],
614 "spaceLength": 10,
615 "stack": false,
616 "steppedLine": true,
617 "targets": [
618 {
619 "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) by (instance, le))",
620 "format": "time_series",
621 "hide": false,
622 "intervalFactor": 2,
623 "legendFormat": "{{instance}} WAL fsync",
624 "metric": "etcd_disk_wal_fsync_duration_seconds_bucket",
625 "refId": "A",
626 "step": 120
627 },
628 {
629 "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) by (instance, le))",
630 "format": "time_series",
631 "intervalFactor": 2,
632 "legendFormat": "{{instance}} DB fsync",
633 "metric": "etcd_disk_backend_commit_duration_seconds_bucket",
634 "refId": "B",
635 "step": 120
636 }
637 ],
638 "thresholds": [],
639 "timeFrom": null,
640 "timeRegions": [],
641 "timeShift": null,
642 "title": "Disk Sync Duration",
643 "tooltip": {
644 "msResolution": false,
645 "shared": true,
646 "sort": 0,
647 "value_type": "cumulative"
648 },
649 "type": "graph",
650 "xaxis": {
651 "buckets": null,
652 "mode": "time",
653 "name": null,
654 "show": true,
655 "values": []
656 },
657 "yaxes": [
658 {
659 "format": "s",
660 "logBase": 1,
661 "max": null,
662 "min": null,
663 "show": true
664 },
665 {
666 "format": "short",
667 "logBase": 1,
668 "max": null,
669 "min": null,
670 "show": false
671 }
672 ],
673 "yaxis": {
674 "align": false,
675 "alignLevel": null
676 }
677 },
678 {
679 "aliasColors": {},
680 "bars": false,
681 "dashLength": 10,
682 "dashes": false,
683 "datasource": "Prometheus",
684 "editable": true,
685 "error": false,
686 "fill": 0,
687 "fillGradient": 0,
688 "gridPos": {
689 "h": 7,
690 "w": 8,
691 "x": 16,
692 "y": 14
693 },
694 "id": 29,
695 "legend": {
696 "avg": false,
697 "current": false,
698 "max": false,
699 "min": false,
700 "show": false,
701 "total": false,
702 "values": false
703 },
704 "lines": true,
705 "linewidth": 2,
706 "links": [],
707 "nullPointMode": "connected",
708 "options": {
709 "dataLinks": []
710 },
711 "percentage": false,
712 "pointradius": 5,
713 "points": false,
714 "renderer": "flot",
715 "seriesOverrides": [],
716 "spaceLength": 10,
717 "stack": false,
718 "steppedLine": false,
719 "targets": [
720 {
721 "expr": "process_resident_memory_bytes",
722 "format": "time_series",
723 "intervalFactor": 2,
724 "legendFormat": "{{instance}} Resident Memory",
725 "metric": "process_resident_memory_bytes",
726 "refId": "A",
727 "step": 120
728 }
729 ],
730 "thresholds": [],
731 "timeFrom": null,
732 "timeRegions": [],
733 "timeShift": null,
734 "title": "Memory",
735 "tooltip": {
736 "msResolution": false,
737 "shared": true,
738 "sort": 0,
739 "value_type": "individual"
740 },
741 "type": "graph",
742 "xaxis": {
743 "buckets": null,
744 "mode": "time",
745 "name": null,
746 "show": true,
747 "values": []
748 },
749 "yaxes": [
750 {
751 "format": "bytes",
752 "label": null,
753 "logBase": 1,
754 "max": null,
755 "min": null,
756 "show": true
757 },
758 {
759 "format": "short",
760 "label": null,
761 "logBase": 1,
762 "max": null,
763 "min": null,
764 "show": true
765 }
766 ],
767 "yaxis": {
768 "align": false,
769 "alignLevel": null
770 }
771 },
772 {
773 "aliasColors": {},
774 "bars": false,
775 "dashLength": 10,
776 "dashes": false,
777 "datasource": "Prometheus",
778 "editable": true,
779 "error": false,
780 "fill": 5,
781 "fillGradient": 0,
782 "gridPos": {
783 "h": 7,
784 "w": 6,
785 "x": 0,
786 "y": 21
787 },
788 "id": 22,
789 "legend": {
790 "avg": false,
791 "current": false,
792 "max": false,
793 "min": false,
794 "show": false,
795 "total": false,
796 "values": false
797 },
798 "lines": true,
799 "linewidth": 2,
800 "links": [],
801 "nullPointMode": "connected",
802 "options": {
803 "dataLinks": []
804 },
805 "percentage": false,
806 "pointradius": 5,
807 "points": false,
808 "renderer": "flot",
809 "seriesOverrides": [],
810 "spaceLength": 10,
811 "stack": true,
812 "steppedLine": false,
813 "targets": [
814 {
815 "expr": "rate(etcd_network_client_grpc_received_bytes_total[5m])",
816 "format": "time_series",
817 "intervalFactor": 2,
818 "legendFormat": "{{instance}} Client Traffic In",
819 "metric": "etcd_network_client_grpc_received_bytes_total",
820 "refId": "A",
821 "step": 120
822 }
823 ],
824 "thresholds": [],
825 "timeFrom": null,
826 "timeRegions": [],
827 "timeShift": null,
828 "title": "Client Traffic In",
829 "tooltip": {
830 "msResolution": false,
831 "shared": true,
832 "sort": 0,
833 "value_type": "individual"
834 },
835 "type": "graph",
836 "xaxis": {
837 "buckets": null,
838 "mode": "time",
839 "name": null,
840 "show": true,
841 "values": []
842 },
843 "yaxes": [
844 {
845 "format": "short",
846 "label": null,
847 "logBase": 1,
848 "max": null,
849 "min": null,
850 "show": true
851 },
852 {
853 "format": "short",
854 "label": null,
855 "logBase": 1,
856 "max": null,
857 "min": null,
858 "show": true
859 }
860 ],
861 "yaxis": {
862 "align": false,
863 "alignLevel": null
864 }
865 },
866 {
867 "aliasColors": {},
868 "bars": false,
869 "dashLength": 10,
870 "dashes": false,
871 "datasource": "Prometheus",
872 "editable": true,
873 "error": false,
874 "fill": 5,
875 "fillGradient": 0,
876 "gridPos": {
877 "h": 7,
878 "w": 6,
879 "x": 6,
880 "y": 21
881 },
882 "id": 21,
883 "legend": {
884 "avg": false,
885 "current": false,
886 "max": false,
887 "min": false,
888 "show": false,
889 "total": false,
890 "values": false
891 },
892 "lines": true,
893 "linewidth": 2,
894 "links": [],
895 "nullPointMode": "connected",
896 "options": {
897 "dataLinks": []
898 },
899 "percentage": false,
900 "pointradius": 5,
901 "points": false,
902 "renderer": "flot",
903 "seriesOverrides": [],
904 "spaceLength": 10,
905 "stack": true,
906 "steppedLine": false,
907 "targets": [
908 {
909 "expr": "rate(etcd_network_client_grpc_sent_bytes_total[5m])",
910 "format": "time_series",
911 "intervalFactor": 2,
912 "legendFormat": "{{instance}} Client Traffic Out",
913 "metric": "etcd_network_client_grpc_sent_bytes_total",
914 "refId": "A",
915 "step": 120
916 }
917 ],
918 "thresholds": [],
919 "timeFrom": null,
920 "timeRegions": [],
921 "timeShift": null,
922 "title": "Client Traffic Out",
923 "tooltip": {
924 "msResolution": false,
925 "shared": true,
926 "sort": 0,
927 "value_type": "individual"
928 },
929 "type": "graph",
930 "xaxis": {
931 "buckets": null,
932 "mode": "time",
933 "name": null,
934 "show": true,
935 "values": []
936 },
937 "yaxes": [
938 {
939 "format": "Bps",
940 "label": null,
941 "logBase": 1,
942 "max": null,
943 "min": null,
944 "show": true
945 },
946 {
947 "format": "short",
948 "label": null,
949 "logBase": 1,
950 "max": null,
951 "min": null,
952 "show": true
953 }
954 ],
955 "yaxis": {
956 "align": false,
957 "alignLevel": null
958 }
959 },
960 {
961 "aliasColors": {},
962 "bars": false,
963 "dashLength": 10,
964 "dashes": false,
965 "datasource": "Prometheus",
966 "editable": true,
967 "error": false,
968 "fill": 0,
969 "fillGradient": 0,
970 "gridPos": {
971 "h": 7,
972 "w": 6,
973 "x": 12,
974 "y": 21
975 },
976 "id": 20,
977 "legend": {
978 "avg": false,
979 "current": false,
980 "max": false,
981 "min": false,
982 "show": false,
983 "total": false,
984 "values": false
985 },
986 "lines": true,
987 "linewidth": 2,
988 "links": [],
989 "nullPointMode": "connected",
990 "options": {
991 "dataLinks": []
992 },
993 "percentage": false,
994 "pointradius": 5,
995 "points": false,
996 "renderer": "flot",
997 "seriesOverrides": [],
998 "spaceLength": 10,
999 "stack": false,
1000 "steppedLine": false,
1001 "targets": [
1002 {
1003 "expr": "sum(rate(etcd_network_peer_received_bytes_total[5m])) by (instance)",
1004 "format": "time_series",
1005 "intervalFactor": 2,
1006 "legendFormat": "{{instance}} Peer Traffic In",
1007 "metric": "etcd_network_peer_received_bytes_total",
1008 "refId": "A",
1009 "step": 120
1010 }
1011 ],
1012 "thresholds": [],
1013 "timeFrom": null,
1014 "timeRegions": [],
1015 "timeShift": null,
1016 "title": "Peer Traffic In",
1017 "tooltip": {
1018 "msResolution": false,
1019 "shared": true,
1020 "sort": 0,
1021 "value_type": "individual"
1022 },
1023 "type": "graph",
1024 "xaxis": {
1025 "buckets": null,
1026 "mode": "time",
1027 "name": null,
1028 "show": true,
1029 "values": []
1030 },
1031 "yaxes": [
1032 {
1033 "format": "Bps",
1034 "label": null,
1035 "logBase": 1,
1036 "max": null,
1037 "min": null,
1038 "show": true
1039 },
1040 {
1041 "format": "short",
1042 "label": null,
1043 "logBase": 1,
1044 "max": null,
1045 "min": null,
1046 "show": true
1047 }
1048 ],
1049 "yaxis": {
1050 "align": false,
1051 "alignLevel": null
1052 }
1053 },
1054 {
1055 "aliasColors": {},
1056 "bars": false,
1057 "dashLength": 10,
1058 "dashes": false,
1059 "datasource": "Prometheus",
1060 "decimals": null,
1061 "editable": true,
1062 "error": false,
1063 "fill": 0,
1064 "fillGradient": 0,
1065 "grid": {},
1066 "gridPos": {
1067 "h": 7,
1068 "w": 6,
1069 "x": 18,
1070 "y": 21
1071 },
1072 "id": 16,
1073 "legend": {
1074 "avg": false,
1075 "current": false,
1076 "max": false,
1077 "min": false,
1078 "show": false,
1079 "total": false,
1080 "values": false
1081 },
1082 "lines": true,
1083 "linewidth": 2,
1084 "links": [],
1085 "nullPointMode": "connected",
1086 "options": {
1087 "dataLinks": []
1088 },
1089 "percentage": false,
1090 "pointradius": 5,
1091 "points": false,
1092 "renderer": "flot",
1093 "seriesOverrides": [],
1094 "spaceLength": 10,
1095 "stack": false,
1096 "steppedLine": false,
1097 "targets": [
1098 {
1099 "expr": "sum(rate(etcd_network_peer_sent_bytes_total[5m])) by (instance)",
1100 "format": "time_series",
1101 "hide": false,
1102 "interval": "",
1103 "intervalFactor": 2,
1104 "legendFormat": "{{instance}} Peer Traffic Out",
1105 "metric": "etcd_network_peer_sent_bytes_total",
1106 "refId": "A",
1107 "step": 120
1108 }
1109 ],
1110 "thresholds": [],
1111 "timeFrom": null,
1112 "timeRegions": [],
1113 "timeShift": null,
1114 "title": "Peer Traffic Out",
1115 "tooltip": {
1116 "msResolution": false,
1117 "shared": true,
1118 "sort": 0,
1119 "value_type": "cumulative"
1120 },
1121 "type": "graph",
1122 "xaxis": {
1123 "buckets": null,
1124 "mode": "time",
1125 "name": null,
1126 "show": true,
1127 "values": []
1128 },
1129 "yaxes": [
1130 {
1131 "format": "Bps",
1132 "logBase": 1,
1133 "max": null,
1134 "min": null,
1135 "show": true
1136 },
1137 {
1138 "format": "short",
1139 "logBase": 1,
1140 "max": null,
1141 "min": null,
1142 "show": true
1143 }
1144 ],
1145 "yaxis": {
1146 "align": false,
1147 "alignLevel": null
1148 }
1149 },
1150 {
1151 "aliasColors": {},
1152 "bars": false,
1153 "dashLength": 10,
1154 "dashes": false,
1155 "datasource": "Prometheus",
1156 "editable": true,
1157 "error": false,
1158 "fill": 0,
1159 "fillGradient": 0,
1160 "gridPos": {
1161 "h": 7,
1162 "w": 12,
1163 "x": 0,
1164 "y": 28
1165 },
1166 "id": 40,
1167 "legend": {
1168 "avg": false,
1169 "current": false,
1170 "max": false,
1171 "min": false,
1172 "show": false,
1173 "total": false,
1174 "values": false
1175 },
1176 "lines": true,
1177 "linewidth": 2,
1178 "links": [],
1179 "nullPointMode": "connected",
1180 "options": {
1181 "dataLinks": []
1182 },
1183 "percentage": false,
1184 "pointradius": 5,
1185 "points": false,
1186 "renderer": "flot",
1187 "seriesOverrides": [],
1188 "spaceLength": 10,
1189 "stack": false,
1190 "steppedLine": false,
1191 "targets": [
1192 {
1193 "expr": "sum(rate(etcd_server_proposals_failed_total[5m]))",
1194 "format": "time_series",
1195 "intervalFactor": 2,
1196 "legendFormat": "Proposal Failure Rate",
1197 "metric": "etcd_server_proposals_failed_total",
1198 "refId": "A",
1199 "step": 60
1200 },
1201 {
1202 "expr": "sum(etcd_server_proposals_pending)",
1203 "format": "time_series",
1204 "intervalFactor": 2,
1205 "legendFormat": "Proposal Pending Total",
1206 "metric": "etcd_server_proposals_pending",
1207 "refId": "B",
1208 "step": 60
1209 },
1210 {
1211 "expr": "sum(rate(etcd_server_proposals_committed_total[5m]))",
1212 "format": "time_series",
1213 "intervalFactor": 2,
1214 "legendFormat": "Proposal Commit Rate",
1215 "metric": "etcd_server_proposals_committed_total",
1216 "refId": "C",
1217 "step": 60
1218 },
1219 {
1220 "expr": "sum(rate(etcd_server_proposals_applied_total[5m]))",
1221 "format": "time_series",
1222 "intervalFactor": 2,
1223 "legendFormat": "Proposal Apply Rate",
1224 "refId": "D",
1225 "step": 60
1226 }
1227 ],
1228 "thresholds": [],
1229 "timeFrom": null,
1230 "timeRegions": [],
1231 "timeShift": null,
1232 "title": "Raft Proposals",
1233 "tooltip": {
1234 "msResolution": false,
1235 "shared": true,
1236 "sort": 0,
1237 "value_type": "individual"
1238 },
1239 "type": "graph",
1240 "xaxis": {
1241 "buckets": null,
1242 "mode": "time",
1243 "name": null,
1244 "show": true,
1245 "values": []
1246 },
1247 "yaxes": [
1248 {
1249 "format": "short",
1250 "label": "",
1251 "logBase": 1,
1252 "max": null,
1253 "min": null,
1254 "show": true
1255 },
1256 {
1257 "format": "short",
1258 "label": null,
1259 "logBase": 1,
1260 "max": null,
1261 "min": null,
1262 "show": true
1263 }
1264 ],
1265 "yaxis": {
1266 "align": false,
1267 "alignLevel": null
1268 }
1269 },
1270 {
1271 "aliasColors": {},
1272 "bars": false,
1273 "dashLength": 10,
1274 "dashes": false,
1275 "datasource": "Prometheus",
1276 "decimals": 0,
1277 "editable": true,
1278 "error": false,
1279 "fill": 0,
1280 "fillGradient": 0,
1281 "gridPos": {
1282 "h": 7,
1283 "w": 12,
1284 "x": 12,
1285 "y": 28
1286 },
1287 "id": 19,
1288 "legend": {
1289 "alignAsTable": false,
1290 "avg": false,
1291 "current": false,
1292 "max": false,
1293 "min": false,
1294 "rightSide": false,
1295 "show": false,
1296 "total": false,
1297 "values": false
1298 },
1299 "lines": true,
1300 "linewidth": 2,
1301 "links": [],
1302 "nullPointMode": "connected",
1303 "options": {
1304 "dataLinks": []
1305 },
1306 "percentage": false,
1307 "pointradius": 5,
1308 "points": false,
1309 "renderer": "flot",
1310 "seriesOverrides": [],
1311 "spaceLength": 10,
1312 "stack": false,
1313 "steppedLine": false,
1314 "targets": [
1315 {
1316 "expr": "changes(etcd_server_leader_changes_seen_total[1d])",
1317 "format": "time_series",
1318 "intervalFactor": 2,
1319 "legendFormat": "{{instance}} Total Leader Elections Per Day",
1320 "metric": "etcd_server_leader_changes_seen_total",
1321 "refId": "A",
1322 "step": 60
1323 }
1324 ],
1325 "thresholds": [],
1326 "timeFrom": null,
1327 "timeRegions": [],
1328 "timeShift": null,
1329 "title": "Total Leader Elections Per Day",
1330 "tooltip": {
1331 "msResolution": false,
1332 "shared": true,
1333 "sort": 0,
1334 "value_type": "individual"
1335 },
1336 "type": "graph",
1337 "xaxis": {
1338 "buckets": null,
1339 "mode": "time",
1340 "name": null,
1341 "show": true,
1342 "values": []
1343 },
1344 "yaxes": [
1345 {
1346 "format": "short",
1347 "label": null,
1348 "logBase": 1,
1349 "max": null,
1350 "min": null,
1351 "show": true
1352 },
1353 {
1354 "format": "short",
1355 "label": null,
1356 "logBase": 1,
1357 "max": null,
1358 "min": null,
1359 "show": true
1360 }
1361 ],
1362 "yaxis": {
1363 "align": false,
1364 "alignLevel": null
1365 }
1366 },
1367 {
1368 "aliasColors": {},
1369 "bars": false,
1370 "dashLength": 10,
1371 "dashes": false,
1372 "datasource": "Prometheus",
1373 "description": "proposals_committed_total records the total number of consensus proposals committed. This gauge should increase over time if the cluster is healthy. Several healthy members of an etcd cluster may have different total committed proposals at once. This discrepancy may be due to recovering from peers after starting, lagging behind the leader, or being the leader and therefore having the most commits. It is important to monitor this metric across all the members in the cluster; a consistently large lag between a single member and its leader indicates that member is slow or unhealthy.\n\nproposals_applied_total records the total number of consensus proposals applied. The etcd server applies every committed proposal asynchronously. The difference between proposals_committed_total and proposals_applied_total should usually be small (within a few thousands even under high load). If the difference between them continues to rise, it indicates that the etcd server is overloaded. This might happen when applying expensive queries like heavy range queries or large txn operations.",
1374 "fill": 1,
1375 "fillGradient": 0,
1376 "gridPos": {
1377 "h": 7,
1378 "w": 12,
1379 "x": 0,
1380 "y": 35
1381 },
1382 "id": 2,
1383 "legend": {
1384 "alignAsTable": true,
1385 "avg": true,
1386 "current": true,
1387 "max": true,
1388 "min": false,
1389 "rightSide": false,
1390 "show": true,
1391 "total": false,
1392 "values": true
1393 },
1394 "lines": true,
1395 "linewidth": 1,
1396 "links": [],
1397 "nullPointMode": "null",
1398 "options": {
1399 "dataLinks": []
1400 },
1401 "percentage": false,
1402 "pointradius": 5,
1403 "points": false,
1404 "renderer": "flot",
1405 "seriesOverrides": [],
1406 "spaceLength": 10,
1407 "stack": false,
1408 "steppedLine": false,
1409 "targets": [
1410 {
1411 "expr": "sum(rate(etcd_server_proposals_committed_total[5m]))",
1412 "format": "time_series",
1413 "intervalFactor": 2,
1414 "legendFormat": "total number of consensus proposals committed",
1415 "metric": "",
1416 "refId": "A",
1417 "step": 60
1418 },
1419 {
1420 "expr": "sum(rate(etcd_server_proposals_applied_total[5m]))",
1421 "format": "time_series",
1422 "intervalFactor": 2,
1423 "legendFormat": "total number of consensus proposals applied",
1424 "metric": "",
1425 "refId": "B",
1426 "step": 60
1427 }
1428 ],
1429 "thresholds": [],
1430 "timeFrom": null,
1431 "timeRegions": [],
1432 "timeShift": null,
1433 "title": "The total number of consensus proposals committed",
1434 "tooltip": {
1435 "shared": true,
1436 "sort": 0,
1437 "value_type": "individual"
1438 },
1439 "type": "graph",
1440 "xaxis": {
1441 "buckets": null,
1442 "mode": "time",
1443 "name": null,
1444 "show": true,
1445 "values": []
1446 },
1447 "yaxes": [
1448 {
1449 "format": "short",
1450 "label": null,
1451 "logBase": 1,
1452 "max": null,
1453 "min": null,
1454 "show": true
1455 },
1456 {
1457 "format": "short",
1458 "label": "",
1459 "logBase": 1,
1460 "max": null,
1461 "min": null,
1462 "show": true
1463 }
1464 ],
1465 "yaxis": {
1466 "align": false,
1467 "alignLevel": null
1468 }
1469 },
1470 {
1471 "aliasColors": {},
1472 "bars": false,
1473 "dashLength": 10,
1474 "dashes": false,
1475 "datasource": "Prometheus",
1476 "description": "indicates how many proposals are queued to commit. Rising pending proposals suggests there is a high client load or the member cannot commit proposals.",
1477 "fill": 1,
1478 "fillGradient": 0,
1479 "gridPos": {
1480 "h": 7,
1481 "w": 12,
1482 "x": 12,
1483 "y": 35
1484 },
1485 "id": 5,
1486 "legend": {
1487 "avg": false,
1488 "current": false,
1489 "max": false,
1490 "min": false,
1491 "show": true,
1492 "total": false,
1493 "values": false
1494 },
1495 "lines": true,
1496 "linewidth": 1,
1497 "links": [],
1498 "nullPointMode": "null",
1499 "options": {
1500 "dataLinks": []
1501 },
1502 "percentage": false,
1503 "pointradius": 5,
1504 "points": false,
1505 "renderer": "flot",
1506 "seriesOverrides": [],
1507 "spaceLength": 10,
1508 "stack": false,
1509 "steppedLine": false,
1510 "targets": [
1511 {
1512 "expr": "sum(etcd_server_proposals_pending)",
1513 "format": "time_series",
1514 "intervalFactor": 2,
1515 "legendFormat": "Proposals pending",
1516 "refId": "A",
1517 "step": 60
1518 }
1519 ],
1520 "thresholds": [],
1521 "timeFrom": null,
1522 "timeRegions": [],
1523 "timeShift": null,
1524 "title": "Proposals pending",
1525 "tooltip": {
1526 "shared": true,
1527 "sort": 0,
1528 "value_type": "individual"
1529 },
1530 "type": "graph",
1531 "xaxis": {
1532 "buckets": null,
1533 "mode": "time",
1534 "name": null,
1535 "show": true,
1536 "values": []
1537 },
1538 "yaxes": [
1539 {
1540 "format": "short",
1541 "label": null,
1542 "logBase": 1,
1543 "max": null,
1544 "min": null,
1545 "show": true
1546 },
1547 {
1548 "format": "short",
1549 "label": null,
1550 "logBase": 1,
1551 "max": null,
1552 "min": null,
1553 "show": true
1554 }
1555 ],
1556 "yaxis": {
1557 "align": false,
1558 "alignLevel": null
1559 }
1560 },
1561 {
1562 "aliasColors": {},
1563 "bars": false,
1564 "dashLength": 10,
1565 "dashes": false,
1566 "datasource": "Prometheus",
1567 "fill": 1,
1568 "fillGradient": 0,
1569 "gridPos": {
1570 "h": 7,
1571 "w": 24,
1572 "x": 0,
1573 "y": 42
1574 },
1575 "id": 7,
1576 "legend": {
1577 "avg": false,
1578 "current": false,
1579 "max": false,
1580 "min": false,
1581 "show": true,
1582 "total": false,
1583 "values": false
1584 },
1585 "lines": true,
1586 "linewidth": 1,
1587 "links": [],
1588 "nullPointMode": "null",
1589 "options": {
1590 "dataLinks": []
1591 },
1592 "percentage": false,
1593 "pointradius": 5,
1594 "points": false,
1595 "renderer": "flot",
1596 "seriesOverrides": [],
1597 "spaceLength": 10,
1598 "stack": false,
1599 "steppedLine": false,
1600 "targets": [
1601 {
1602 "expr": "sum(rate(etcd_disk_wal_fsync_duration_seconds_sum[1m]))",
1603 "format": "time_series",
1604 "intervalFactor": 2,
1605 "legendFormat": "\tThe latency distributions of fsync called by wal",
1606 "refId": "A",
1607 "step": 30
1608 },
1609 {
1610 "expr": "sum(rate(etcd_disk_backend_commit_duration_seconds_sum[1m]))",
1611 "format": "time_series",
1612 "intervalFactor": 2,
1613 "legendFormat": "The latency distributions of commit called by backend",
1614 "refId": "B",
1615 "step": 30
1616 }
1617 ],
1618 "thresholds": [],
1619 "timeFrom": null,
1620 "timeRegions": [],
1621 "timeShift": null,
1622 "title": "Disks operations",
1623 "tooltip": {
1624 "shared": true,
1625 "sort": 0,
1626 "value_type": "individual"
1627 },
1628 "type": "graph",
1629 "xaxis": {
1630 "buckets": null,
1631 "mode": "time",
1632 "name": null,
1633 "show": true,
1634 "values": []
1635 },
1636 "yaxes": [
1637 {
1638 "format": "short",
1639 "label": null,
1640 "logBase": 1,
1641 "max": null,
1642 "min": null,
1643 "show": true
1644 },
1645 {
1646 "format": "short",
1647 "label": null,
1648 "logBase": 1,
1649 "max": null,
1650 "min": null,
1651 "show": true
1652 }
1653 ],
1654 "yaxis": {
1655 "align": false,
1656 "alignLevel": null
1657 }
1658 },
1659 {
1660 "aliasColors": {},
1661 "bars": false,
1662 "dashLength": 10,
1663 "dashes": false,
1664 "datasource": "Prometheus",
1665 "fill": 1,
1666 "fillGradient": 0,
1667 "gridPos": {
1668 "h": 7,
1669 "w": 24,
1670 "x": 0,
1671 "y": 49
1672 },
1673 "id": 8,
1674 "legend": {
1675 "avg": false,
1676 "current": false,
1677 "max": false,
1678 "min": false,
1679 "show": true,
1680 "total": false,
1681 "values": false
1682 },
1683 "lines": true,
1684 "linewidth": 1,
1685 "links": [],
1686 "nullPointMode": "null",
1687 "options": {
1688 "dataLinks": []
1689 },
1690 "percentage": false,
1691 "pointradius": 5,
1692 "points": false,
1693 "renderer": "flot",
1694 "seriesOverrides": [],
1695 "spaceLength": 10,
1696 "stack": false,
1697 "steppedLine": false,
1698 "targets": [
1699 {
1700 "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total[1m]))",
1701 "format": "time_series",
1702 "intervalFactor": 2,
1703 "legendFormat": "The total number of bytes received by grpc clients",
1704 "refId": "A",
1705 "step": 30
1706 },
1707 {
1708 "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total[1m]))",
1709 "format": "time_series",
1710 "intervalFactor": 2,
1711 "legendFormat": "The total number of bytes sent to grpc clients",
1712 "refId": "B",
1713 "step": 30
1714 }
1715 ],
1716 "thresholds": [],
1717 "timeFrom": null,
1718 "timeRegions": [],
1719 "timeShift": null,
1720 "title": "Network",
1721 "tooltip": {
1722 "shared": true,
1723 "sort": 0,
1724 "value_type": "individual"
1725 },
1726 "type": "graph",
1727 "xaxis": {
1728 "buckets": null,
1729 "mode": "time",
1730 "name": null,
1731 "show": true,
1732 "values": []
1733 },
1734 "yaxes": [
1735 {
1736 "format": "short",
1737 "label": null,
1738 "logBase": 1,
1739 "max": null,
1740 "min": null,
1741 "show": true
1742 },
1743 {
1744 "format": "short",
1745 "label": null,
1746 "logBase": 1,
1747 "max": null,
1748 "min": null,
1749 "show": true
1750 }
1751 ],
1752 "yaxis": {
1753 "align": false,
1754 "alignLevel": null
1755 }
1756 },
1757 {
1758 "aliasColors": {},
1759 "bars": false,
1760 "dashLength": 10,
1761 "dashes": false,
1762 "datasource": "Prometheus",
1763 "description": "Abnormally high snapshot duration (snapshot_save_total_duration_seconds) indicates disk issues and might cause the cluster to be unstable.",
1764 "fill": 1,
1765 "fillGradient": 0,
1766 "gridPos": {
1767 "h": 7,
1768 "w": 24,
1769 "x": 0,
1770 "y": 56
1771 },
1772 "id": 9,
1773 "legend": {
1774 "avg": false,
1775 "current": false,
1776 "max": false,
1777 "min": false,
1778 "show": true,
1779 "total": false,
1780 "values": false
1781 },
1782 "lines": true,
1783 "linewidth": 1,
1784 "links": [],
1785 "nullPointMode": "null",
1786 "options": {
1787 "dataLinks": []
1788 },
1789 "percentage": false,
1790 "pointradius": 5,
1791 "points": false,
1792 "renderer": "flot",
1793 "seriesOverrides": [],
1794 "spaceLength": 10,
1795 "stack": false,
1796 "steppedLine": false,
1797 "targets": [
1798 {
1799 "expr": "sum(rate(etcd_debugging_snap_save_total_duration_seconds_sum[1m]))",
1800 "format": "time_series",
1801 "intervalFactor": 2,
1802 "legendFormat": "The total latency distributions of save called by snapshot",
1803 "refId": "A",
1804 "step": 30
1805 }
1806 ],
1807 "thresholds": [],
1808 "timeFrom": null,
1809 "timeRegions": [],
1810 "timeShift": null,
1811 "title": "Snapshot duration",
1812 "tooltip": {
1813 "shared": true,
1814 "sort": 0,
1815 "value_type": "individual"
1816 },
1817 "type": "graph",
1818 "xaxis": {
1819 "buckets": null,
1820 "mode": "time",
1821 "name": null,
1822 "show": true,
1823 "values": []
1824 },
1825 "yaxes": [
1826 {
1827 "format": "short",
1828 "label": null,
1829 "logBase": 1,
1830 "max": null,
1831 "min": null,
1832 "show": true
1833 },
1834 {
1835 "format": "short",
1836 "label": null,
1837 "logBase": 1,
1838 "max": null,
1839 "min": null,
1840 "show": true
1841 }
1842 ],
1843 "yaxis": {
1844 "align": false,
1845 "alignLevel": null
1846 }
1847 }
1848 ],
1849 "schemaVersion": 20,
1850 "style": "dark",
1851 "tags": [
1852 "nem-monitoring"
1853 ],
1854 "templating": {
1855 "list": []
1856 },
1857 "time": {
1858 "from": "now-6h",
1859 "to": "now"
1860 },
1861 "timepicker": {
1862 "refresh_intervals": [
1863 "5s",
1864 "10s",
1865 "30s",
1866 "1m",
1867 "5m",
1868 "15m",
1869 "30m",
1870 "1h",
1871 "2h",
1872 "1d"
1873 ],
1874 "time_options": [
1875 "5m",
1876 "15m",
1877 "1h",
1878 "6h",
1879 "12h",
1880 "24h",
1881 "2d",
1882 "7d",
1883 "30d"
1884 ]
1885 },
1886 "timezone": "browser",
1887 "title": "ETCD Metrics",
1888 "uid": "oS7fwgzMz",
1889 "version": 2
1890}