Holger Hildebrandt | fa07499 | 2020-03-27 15:42:06 +0000 | [diff] [blame] | 1 | package metrics |
| 2 | |
| 3 | import ( |
| 4 | "runtime" |
| 5 | "strings" |
| 6 | "time" |
| 7 | |
| 8 | "github.com/hashicorp/go-immutable-radix" |
| 9 | ) |
| 10 | |
| 11 | type Label struct { |
| 12 | Name string |
| 13 | Value string |
| 14 | } |
| 15 | |
| 16 | func (m *Metrics) SetGauge(key []string, val float32) { |
| 17 | m.SetGaugeWithLabels(key, val, nil) |
| 18 | } |
| 19 | |
| 20 | func (m *Metrics) SetGaugeWithLabels(key []string, val float32, labels []Label) { |
| 21 | if m.HostName != "" { |
| 22 | if m.EnableHostnameLabel { |
| 23 | labels = append(labels, Label{"host", m.HostName}) |
| 24 | } else if m.EnableHostname { |
| 25 | key = insert(0, m.HostName, key) |
| 26 | } |
| 27 | } |
| 28 | if m.EnableTypePrefix { |
| 29 | key = insert(0, "gauge", key) |
| 30 | } |
| 31 | if m.ServiceName != "" { |
| 32 | if m.EnableServiceLabel { |
| 33 | labels = append(labels, Label{"service", m.ServiceName}) |
| 34 | } else { |
| 35 | key = insert(0, m.ServiceName, key) |
| 36 | } |
| 37 | } |
| 38 | allowed, labelsFiltered := m.allowMetric(key, labels) |
| 39 | if !allowed { |
| 40 | return |
| 41 | } |
| 42 | m.sink.SetGaugeWithLabels(key, val, labelsFiltered) |
| 43 | } |
| 44 | |
| 45 | func (m *Metrics) EmitKey(key []string, val float32) { |
| 46 | if m.EnableTypePrefix { |
| 47 | key = insert(0, "kv", key) |
| 48 | } |
| 49 | if m.ServiceName != "" { |
| 50 | key = insert(0, m.ServiceName, key) |
| 51 | } |
| 52 | allowed, _ := m.allowMetric(key, nil) |
| 53 | if !allowed { |
| 54 | return |
| 55 | } |
| 56 | m.sink.EmitKey(key, val) |
| 57 | } |
| 58 | |
| 59 | func (m *Metrics) IncrCounter(key []string, val float32) { |
| 60 | m.IncrCounterWithLabels(key, val, nil) |
| 61 | } |
| 62 | |
| 63 | func (m *Metrics) IncrCounterWithLabels(key []string, val float32, labels []Label) { |
| 64 | if m.HostName != "" && m.EnableHostnameLabel { |
| 65 | labels = append(labels, Label{"host", m.HostName}) |
| 66 | } |
| 67 | if m.EnableTypePrefix { |
| 68 | key = insert(0, "counter", key) |
| 69 | } |
| 70 | if m.ServiceName != "" { |
| 71 | if m.EnableServiceLabel { |
| 72 | labels = append(labels, Label{"service", m.ServiceName}) |
| 73 | } else { |
| 74 | key = insert(0, m.ServiceName, key) |
| 75 | } |
| 76 | } |
| 77 | allowed, labelsFiltered := m.allowMetric(key, labels) |
| 78 | if !allowed { |
| 79 | return |
| 80 | } |
| 81 | m.sink.IncrCounterWithLabels(key, val, labelsFiltered) |
| 82 | } |
| 83 | |
| 84 | func (m *Metrics) AddSample(key []string, val float32) { |
| 85 | m.AddSampleWithLabels(key, val, nil) |
| 86 | } |
| 87 | |
| 88 | func (m *Metrics) AddSampleWithLabels(key []string, val float32, labels []Label) { |
| 89 | if m.HostName != "" && m.EnableHostnameLabel { |
| 90 | labels = append(labels, Label{"host", m.HostName}) |
| 91 | } |
| 92 | if m.EnableTypePrefix { |
| 93 | key = insert(0, "sample", key) |
| 94 | } |
| 95 | if m.ServiceName != "" { |
| 96 | if m.EnableServiceLabel { |
| 97 | labels = append(labels, Label{"service", m.ServiceName}) |
| 98 | } else { |
| 99 | key = insert(0, m.ServiceName, key) |
| 100 | } |
| 101 | } |
| 102 | allowed, labelsFiltered := m.allowMetric(key, labels) |
| 103 | if !allowed { |
| 104 | return |
| 105 | } |
| 106 | m.sink.AddSampleWithLabels(key, val, labelsFiltered) |
| 107 | } |
| 108 | |
| 109 | func (m *Metrics) MeasureSince(key []string, start time.Time) { |
| 110 | m.MeasureSinceWithLabels(key, start, nil) |
| 111 | } |
| 112 | |
| 113 | func (m *Metrics) MeasureSinceWithLabels(key []string, start time.Time, labels []Label) { |
| 114 | if m.HostName != "" && m.EnableHostnameLabel { |
| 115 | labels = append(labels, Label{"host", m.HostName}) |
| 116 | } |
| 117 | if m.EnableTypePrefix { |
| 118 | key = insert(0, "timer", key) |
| 119 | } |
| 120 | if m.ServiceName != "" { |
| 121 | if m.EnableServiceLabel { |
| 122 | labels = append(labels, Label{"service", m.ServiceName}) |
| 123 | } else { |
| 124 | key = insert(0, m.ServiceName, key) |
| 125 | } |
| 126 | } |
| 127 | allowed, labelsFiltered := m.allowMetric(key, labels) |
| 128 | if !allowed { |
| 129 | return |
| 130 | } |
| 131 | now := time.Now() |
| 132 | elapsed := now.Sub(start) |
| 133 | msec := float32(elapsed.Nanoseconds()) / float32(m.TimerGranularity) |
| 134 | m.sink.AddSampleWithLabels(key, msec, labelsFiltered) |
| 135 | } |
| 136 | |
| 137 | // UpdateFilter overwrites the existing filter with the given rules. |
| 138 | func (m *Metrics) UpdateFilter(allow, block []string) { |
| 139 | m.UpdateFilterAndLabels(allow, block, m.AllowedLabels, m.BlockedLabels) |
| 140 | } |
| 141 | |
| 142 | // UpdateFilterAndLabels overwrites the existing filter with the given rules. |
| 143 | func (m *Metrics) UpdateFilterAndLabels(allow, block, allowedLabels, blockedLabels []string) { |
| 144 | m.filterLock.Lock() |
| 145 | defer m.filterLock.Unlock() |
| 146 | |
| 147 | m.AllowedPrefixes = allow |
| 148 | m.BlockedPrefixes = block |
| 149 | |
| 150 | if allowedLabels == nil { |
| 151 | // Having a white list means we take only elements from it |
| 152 | m.allowedLabels = nil |
| 153 | } else { |
| 154 | m.allowedLabels = make(map[string]bool) |
| 155 | for _, v := range allowedLabels { |
| 156 | m.allowedLabels[v] = true |
| 157 | } |
| 158 | } |
| 159 | m.blockedLabels = make(map[string]bool) |
| 160 | for _, v := range blockedLabels { |
| 161 | m.blockedLabels[v] = true |
| 162 | } |
| 163 | m.AllowedLabels = allowedLabels |
| 164 | m.BlockedLabels = blockedLabels |
| 165 | |
| 166 | m.filter = iradix.New() |
| 167 | for _, prefix := range m.AllowedPrefixes { |
| 168 | m.filter, _, _ = m.filter.Insert([]byte(prefix), true) |
| 169 | } |
| 170 | for _, prefix := range m.BlockedPrefixes { |
| 171 | m.filter, _, _ = m.filter.Insert([]byte(prefix), false) |
| 172 | } |
| 173 | } |
| 174 | |
| 175 | // labelIsAllowed return true if a should be included in metric |
| 176 | // the caller should lock m.filterLock while calling this method |
| 177 | func (m *Metrics) labelIsAllowed(label *Label) bool { |
| 178 | labelName := (*label).Name |
| 179 | if m.blockedLabels != nil { |
| 180 | _, ok := m.blockedLabels[labelName] |
| 181 | if ok { |
| 182 | // If present, let's remove this label |
| 183 | return false |
| 184 | } |
| 185 | } |
| 186 | if m.allowedLabels != nil { |
| 187 | _, ok := m.allowedLabels[labelName] |
| 188 | return ok |
| 189 | } |
| 190 | // Allow by default |
| 191 | return true |
| 192 | } |
| 193 | |
| 194 | // filterLabels return only allowed labels |
| 195 | // the caller should lock m.filterLock while calling this method |
| 196 | func (m *Metrics) filterLabels(labels []Label) []Label { |
| 197 | if labels == nil { |
| 198 | return nil |
| 199 | } |
| 200 | toReturn := []Label{} |
| 201 | for _, label := range labels { |
| 202 | if m.labelIsAllowed(&label) { |
| 203 | toReturn = append(toReturn, label) |
| 204 | } |
| 205 | } |
| 206 | return toReturn |
| 207 | } |
| 208 | |
| 209 | // Returns whether the metric should be allowed based on configured prefix filters |
| 210 | // Also return the applicable labels |
| 211 | func (m *Metrics) allowMetric(key []string, labels []Label) (bool, []Label) { |
| 212 | m.filterLock.RLock() |
| 213 | defer m.filterLock.RUnlock() |
| 214 | |
| 215 | if m.filter == nil || m.filter.Len() == 0 { |
| 216 | return m.Config.FilterDefault, m.filterLabels(labels) |
| 217 | } |
| 218 | |
| 219 | _, allowed, ok := m.filter.Root().LongestPrefix([]byte(strings.Join(key, "."))) |
| 220 | if !ok { |
| 221 | return m.Config.FilterDefault, m.filterLabels(labels) |
| 222 | } |
| 223 | |
| 224 | return allowed.(bool), m.filterLabels(labels) |
| 225 | } |
| 226 | |
| 227 | // Periodically collects runtime stats to publish |
| 228 | func (m *Metrics) collectStats() { |
| 229 | for { |
| 230 | time.Sleep(m.ProfileInterval) |
| 231 | m.emitRuntimeStats() |
| 232 | } |
| 233 | } |
| 234 | |
| 235 | // Emits various runtime statsitics |
| 236 | func (m *Metrics) emitRuntimeStats() { |
| 237 | // Export number of Goroutines |
| 238 | numRoutines := runtime.NumGoroutine() |
| 239 | m.SetGauge([]string{"runtime", "num_goroutines"}, float32(numRoutines)) |
| 240 | |
| 241 | // Export memory stats |
| 242 | var stats runtime.MemStats |
| 243 | runtime.ReadMemStats(&stats) |
| 244 | m.SetGauge([]string{"runtime", "alloc_bytes"}, float32(stats.Alloc)) |
| 245 | m.SetGauge([]string{"runtime", "sys_bytes"}, float32(stats.Sys)) |
| 246 | m.SetGauge([]string{"runtime", "malloc_count"}, float32(stats.Mallocs)) |
| 247 | m.SetGauge([]string{"runtime", "free_count"}, float32(stats.Frees)) |
| 248 | m.SetGauge([]string{"runtime", "heap_objects"}, float32(stats.HeapObjects)) |
| 249 | m.SetGauge([]string{"runtime", "total_gc_pause_ns"}, float32(stats.PauseTotalNs)) |
| 250 | m.SetGauge([]string{"runtime", "total_gc_runs"}, float32(stats.NumGC)) |
| 251 | |
| 252 | // Export info about the last few GC runs |
| 253 | num := stats.NumGC |
| 254 | |
| 255 | // Handle wrap around |
| 256 | if num < m.lastNumGC { |
| 257 | m.lastNumGC = 0 |
| 258 | } |
| 259 | |
| 260 | // Ensure we don't scan more than 256 |
| 261 | if num-m.lastNumGC >= 256 { |
| 262 | m.lastNumGC = num - 255 |
| 263 | } |
| 264 | |
| 265 | for i := m.lastNumGC; i < num; i++ { |
| 266 | pause := stats.PauseNs[i%256] |
| 267 | m.AddSample([]string{"runtime", "gc_pause_ns"}, float32(pause)) |
| 268 | } |
| 269 | m.lastNumGC = num |
| 270 | } |
| 271 | |
| 272 | // Inserts a string value at an index into the slice |
| 273 | func insert(i int, v string, s []string) []string { |
| 274 | s = append(s, "") |
| 275 | copy(s[i+1:], s[i:]) |
| 276 | s[i] = v |
| 277 | return s |
| 278 | } |