| Scott Baker | ed4efab | 2020-01-13 19:12:25 -0800 | [diff] [blame] | 1 | package metrics |
| 2 | |
| 3 | import ( |
| 4 | "runtime" |
| 5 | "runtime/pprof" |
| 6 | "time" |
| 7 | ) |
| 8 | |
| 9 | var ( |
| 10 | memStats runtime.MemStats |
| 11 | runtimeMetrics struct { |
| 12 | MemStats struct { |
| 13 | Alloc Gauge |
| 14 | BuckHashSys Gauge |
| 15 | DebugGC Gauge |
| 16 | EnableGC Gauge |
| 17 | Frees Gauge |
| 18 | HeapAlloc Gauge |
| 19 | HeapIdle Gauge |
| 20 | HeapInuse Gauge |
| 21 | HeapObjects Gauge |
| 22 | HeapReleased Gauge |
| 23 | HeapSys Gauge |
| 24 | LastGC Gauge |
| 25 | Lookups Gauge |
| 26 | Mallocs Gauge |
| 27 | MCacheInuse Gauge |
| 28 | MCacheSys Gauge |
| 29 | MSpanInuse Gauge |
| 30 | MSpanSys Gauge |
| 31 | NextGC Gauge |
| 32 | NumGC Gauge |
| 33 | GCCPUFraction GaugeFloat64 |
| 34 | PauseNs Histogram |
| 35 | PauseTotalNs Gauge |
| 36 | StackInuse Gauge |
| 37 | StackSys Gauge |
| 38 | Sys Gauge |
| 39 | TotalAlloc Gauge |
| 40 | } |
| 41 | NumCgoCall Gauge |
| 42 | NumGoroutine Gauge |
| 43 | NumThread Gauge |
| 44 | ReadMemStats Timer |
| 45 | } |
| 46 | frees uint64 |
| 47 | lookups uint64 |
| 48 | mallocs uint64 |
| 49 | numGC uint32 |
| 50 | numCgoCalls int64 |
| 51 | |
| 52 | threadCreateProfile = pprof.Lookup("threadcreate") |
| 53 | ) |
| 54 | |
| 55 | // Capture new values for the Go runtime statistics exported in |
| 56 | // runtime.MemStats. This is designed to be called as a goroutine. |
| 57 | func CaptureRuntimeMemStats(r Registry, d time.Duration) { |
| 58 | for _ = range time.Tick(d) { |
| 59 | CaptureRuntimeMemStatsOnce(r) |
| 60 | } |
| 61 | } |
| 62 | |
| 63 | // Capture new values for the Go runtime statistics exported in |
| 64 | // runtime.MemStats. This is designed to be called in a background |
| 65 | // goroutine. Giving a registry which has not been given to |
| 66 | // RegisterRuntimeMemStats will panic. |
| 67 | // |
| 68 | // Be very careful with this because runtime.ReadMemStats calls the C |
| 69 | // functions runtime·semacquire(&runtime·worldsema) and runtime·stoptheworld() |
| 70 | // and that last one does what it says on the tin. |
| 71 | func CaptureRuntimeMemStatsOnce(r Registry) { |
| 72 | t := time.Now() |
| 73 | runtime.ReadMemStats(&memStats) // This takes 50-200us. |
| 74 | runtimeMetrics.ReadMemStats.UpdateSince(t) |
| 75 | |
| 76 | runtimeMetrics.MemStats.Alloc.Update(int64(memStats.Alloc)) |
| 77 | runtimeMetrics.MemStats.BuckHashSys.Update(int64(memStats.BuckHashSys)) |
| 78 | if memStats.DebugGC { |
| 79 | runtimeMetrics.MemStats.DebugGC.Update(1) |
| 80 | } else { |
| 81 | runtimeMetrics.MemStats.DebugGC.Update(0) |
| 82 | } |
| 83 | if memStats.EnableGC { |
| 84 | runtimeMetrics.MemStats.EnableGC.Update(1) |
| 85 | } else { |
| 86 | runtimeMetrics.MemStats.EnableGC.Update(0) |
| 87 | } |
| 88 | |
| 89 | runtimeMetrics.MemStats.Frees.Update(int64(memStats.Frees - frees)) |
| 90 | runtimeMetrics.MemStats.HeapAlloc.Update(int64(memStats.HeapAlloc)) |
| 91 | runtimeMetrics.MemStats.HeapIdle.Update(int64(memStats.HeapIdle)) |
| 92 | runtimeMetrics.MemStats.HeapInuse.Update(int64(memStats.HeapInuse)) |
| 93 | runtimeMetrics.MemStats.HeapObjects.Update(int64(memStats.HeapObjects)) |
| 94 | runtimeMetrics.MemStats.HeapReleased.Update(int64(memStats.HeapReleased)) |
| 95 | runtimeMetrics.MemStats.HeapSys.Update(int64(memStats.HeapSys)) |
| 96 | runtimeMetrics.MemStats.LastGC.Update(int64(memStats.LastGC)) |
| 97 | runtimeMetrics.MemStats.Lookups.Update(int64(memStats.Lookups - lookups)) |
| 98 | runtimeMetrics.MemStats.Mallocs.Update(int64(memStats.Mallocs - mallocs)) |
| 99 | runtimeMetrics.MemStats.MCacheInuse.Update(int64(memStats.MCacheInuse)) |
| 100 | runtimeMetrics.MemStats.MCacheSys.Update(int64(memStats.MCacheSys)) |
| 101 | runtimeMetrics.MemStats.MSpanInuse.Update(int64(memStats.MSpanInuse)) |
| 102 | runtimeMetrics.MemStats.MSpanSys.Update(int64(memStats.MSpanSys)) |
| 103 | runtimeMetrics.MemStats.NextGC.Update(int64(memStats.NextGC)) |
| 104 | runtimeMetrics.MemStats.NumGC.Update(int64(memStats.NumGC - numGC)) |
| 105 | runtimeMetrics.MemStats.GCCPUFraction.Update(gcCPUFraction(&memStats)) |
| 106 | |
| 107 | // <https://code.google.com/p/go/source/browse/src/pkg/runtime/mgc0.c> |
| 108 | i := numGC % uint32(len(memStats.PauseNs)) |
| 109 | ii := memStats.NumGC % uint32(len(memStats.PauseNs)) |
| 110 | if memStats.NumGC-numGC >= uint32(len(memStats.PauseNs)) { |
| 111 | for i = 0; i < uint32(len(memStats.PauseNs)); i++ { |
| 112 | runtimeMetrics.MemStats.PauseNs.Update(int64(memStats.PauseNs[i])) |
| 113 | } |
| 114 | } else { |
| 115 | if i > ii { |
| 116 | for ; i < uint32(len(memStats.PauseNs)); i++ { |
| 117 | runtimeMetrics.MemStats.PauseNs.Update(int64(memStats.PauseNs[i])) |
| 118 | } |
| 119 | i = 0 |
| 120 | } |
| 121 | for ; i < ii; i++ { |
| 122 | runtimeMetrics.MemStats.PauseNs.Update(int64(memStats.PauseNs[i])) |
| 123 | } |
| 124 | } |
| 125 | frees = memStats.Frees |
| 126 | lookups = memStats.Lookups |
| 127 | mallocs = memStats.Mallocs |
| 128 | numGC = memStats.NumGC |
| 129 | |
| 130 | runtimeMetrics.MemStats.PauseTotalNs.Update(int64(memStats.PauseTotalNs)) |
| 131 | runtimeMetrics.MemStats.StackInuse.Update(int64(memStats.StackInuse)) |
| 132 | runtimeMetrics.MemStats.StackSys.Update(int64(memStats.StackSys)) |
| 133 | runtimeMetrics.MemStats.Sys.Update(int64(memStats.Sys)) |
| 134 | runtimeMetrics.MemStats.TotalAlloc.Update(int64(memStats.TotalAlloc)) |
| 135 | |
| 136 | currentNumCgoCalls := numCgoCall() |
| 137 | runtimeMetrics.NumCgoCall.Update(currentNumCgoCalls - numCgoCalls) |
| 138 | numCgoCalls = currentNumCgoCalls |
| 139 | |
| 140 | runtimeMetrics.NumGoroutine.Update(int64(runtime.NumGoroutine())) |
| 141 | |
| 142 | runtimeMetrics.NumThread.Update(int64(threadCreateProfile.Count())) |
| 143 | } |
| 144 | |
| 145 | // Register runtimeMetrics for the Go runtime statistics exported in runtime and |
| 146 | // specifically runtime.MemStats. The runtimeMetrics are named by their |
| 147 | // fully-qualified Go symbols, i.e. runtime.MemStats.Alloc. |
| 148 | func RegisterRuntimeMemStats(r Registry) { |
| 149 | runtimeMetrics.MemStats.Alloc = NewGauge() |
| 150 | runtimeMetrics.MemStats.BuckHashSys = NewGauge() |
| 151 | runtimeMetrics.MemStats.DebugGC = NewGauge() |
| 152 | runtimeMetrics.MemStats.EnableGC = NewGauge() |
| 153 | runtimeMetrics.MemStats.Frees = NewGauge() |
| 154 | runtimeMetrics.MemStats.HeapAlloc = NewGauge() |
| 155 | runtimeMetrics.MemStats.HeapIdle = NewGauge() |
| 156 | runtimeMetrics.MemStats.HeapInuse = NewGauge() |
| 157 | runtimeMetrics.MemStats.HeapObjects = NewGauge() |
| 158 | runtimeMetrics.MemStats.HeapReleased = NewGauge() |
| 159 | runtimeMetrics.MemStats.HeapSys = NewGauge() |
| 160 | runtimeMetrics.MemStats.LastGC = NewGauge() |
| 161 | runtimeMetrics.MemStats.Lookups = NewGauge() |
| 162 | runtimeMetrics.MemStats.Mallocs = NewGauge() |
| 163 | runtimeMetrics.MemStats.MCacheInuse = NewGauge() |
| 164 | runtimeMetrics.MemStats.MCacheSys = NewGauge() |
| 165 | runtimeMetrics.MemStats.MSpanInuse = NewGauge() |
| 166 | runtimeMetrics.MemStats.MSpanSys = NewGauge() |
| 167 | runtimeMetrics.MemStats.NextGC = NewGauge() |
| 168 | runtimeMetrics.MemStats.NumGC = NewGauge() |
| 169 | runtimeMetrics.MemStats.GCCPUFraction = NewGaugeFloat64() |
| 170 | runtimeMetrics.MemStats.PauseNs = NewHistogram(NewExpDecaySample(1028, 0.015)) |
| 171 | runtimeMetrics.MemStats.PauseTotalNs = NewGauge() |
| 172 | runtimeMetrics.MemStats.StackInuse = NewGauge() |
| 173 | runtimeMetrics.MemStats.StackSys = NewGauge() |
| 174 | runtimeMetrics.MemStats.Sys = NewGauge() |
| 175 | runtimeMetrics.MemStats.TotalAlloc = NewGauge() |
| 176 | runtimeMetrics.NumCgoCall = NewGauge() |
| 177 | runtimeMetrics.NumGoroutine = NewGauge() |
| 178 | runtimeMetrics.NumThread = NewGauge() |
| 179 | runtimeMetrics.ReadMemStats = NewTimer() |
| 180 | |
| 181 | r.Register("runtime.MemStats.Alloc", runtimeMetrics.MemStats.Alloc) |
| 182 | r.Register("runtime.MemStats.BuckHashSys", runtimeMetrics.MemStats.BuckHashSys) |
| 183 | r.Register("runtime.MemStats.DebugGC", runtimeMetrics.MemStats.DebugGC) |
| 184 | r.Register("runtime.MemStats.EnableGC", runtimeMetrics.MemStats.EnableGC) |
| 185 | r.Register("runtime.MemStats.Frees", runtimeMetrics.MemStats.Frees) |
| 186 | r.Register("runtime.MemStats.HeapAlloc", runtimeMetrics.MemStats.HeapAlloc) |
| 187 | r.Register("runtime.MemStats.HeapIdle", runtimeMetrics.MemStats.HeapIdle) |
| 188 | r.Register("runtime.MemStats.HeapInuse", runtimeMetrics.MemStats.HeapInuse) |
| 189 | r.Register("runtime.MemStats.HeapObjects", runtimeMetrics.MemStats.HeapObjects) |
| 190 | r.Register("runtime.MemStats.HeapReleased", runtimeMetrics.MemStats.HeapReleased) |
| 191 | r.Register("runtime.MemStats.HeapSys", runtimeMetrics.MemStats.HeapSys) |
| 192 | r.Register("runtime.MemStats.LastGC", runtimeMetrics.MemStats.LastGC) |
| 193 | r.Register("runtime.MemStats.Lookups", runtimeMetrics.MemStats.Lookups) |
| 194 | r.Register("runtime.MemStats.Mallocs", runtimeMetrics.MemStats.Mallocs) |
| 195 | r.Register("runtime.MemStats.MCacheInuse", runtimeMetrics.MemStats.MCacheInuse) |
| 196 | r.Register("runtime.MemStats.MCacheSys", runtimeMetrics.MemStats.MCacheSys) |
| 197 | r.Register("runtime.MemStats.MSpanInuse", runtimeMetrics.MemStats.MSpanInuse) |
| 198 | r.Register("runtime.MemStats.MSpanSys", runtimeMetrics.MemStats.MSpanSys) |
| 199 | r.Register("runtime.MemStats.NextGC", runtimeMetrics.MemStats.NextGC) |
| 200 | r.Register("runtime.MemStats.NumGC", runtimeMetrics.MemStats.NumGC) |
| 201 | r.Register("runtime.MemStats.GCCPUFraction", runtimeMetrics.MemStats.GCCPUFraction) |
| 202 | r.Register("runtime.MemStats.PauseNs", runtimeMetrics.MemStats.PauseNs) |
| 203 | r.Register("runtime.MemStats.PauseTotalNs", runtimeMetrics.MemStats.PauseTotalNs) |
| 204 | r.Register("runtime.MemStats.StackInuse", runtimeMetrics.MemStats.StackInuse) |
| 205 | r.Register("runtime.MemStats.StackSys", runtimeMetrics.MemStats.StackSys) |
| 206 | r.Register("runtime.MemStats.Sys", runtimeMetrics.MemStats.Sys) |
| 207 | r.Register("runtime.MemStats.TotalAlloc", runtimeMetrics.MemStats.TotalAlloc) |
| 208 | r.Register("runtime.NumCgoCall", runtimeMetrics.NumCgoCall) |
| 209 | r.Register("runtime.NumGoroutine", runtimeMetrics.NumGoroutine) |
| 210 | r.Register("runtime.NumThread", runtimeMetrics.NumThread) |
| 211 | r.Register("runtime.ReadMemStats", runtimeMetrics.ReadMemStats) |
| 212 | } |