Matteo Scandolo | 3ed8987 | 2020-07-15 17:01:02 -0700 | [diff] [blame] | 1 | # Copyright 2017-present Open Networking Foundation |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. |
| 14 | |
| 15 | # This tool collects CPU and Memory informations for each container in the VOLTHA stack |
| 16 | |
| 17 | # NOTE |
| 18 | # Collecting the info for all containers in the same chart can be confusing, |
| 19 | # we may want to create subcharts for the different groups, eg: infra, ONOS, core, adapters |
Matteo Scandolo | 984cf7b | 2020-07-29 13:54:20 -0700 | [diff] [blame^] | 20 | import csv |
| 21 | from sys import platform as sys_pf |
| 22 | |
| 23 | if sys_pf == 'darwin': |
| 24 | import matplotlib |
| 25 | |
| 26 | matplotlib.use("TkAgg") |
Matteo Scandolo | 3ed8987 | 2020-07-15 17:01:02 -0700 | [diff] [blame] | 27 | |
| 28 | import argparse |
| 29 | import requests |
| 30 | import matplotlib.pyplot as plt |
| 31 | import matplotlib.dates as mdates |
| 32 | from datetime import datetime |
| 33 | import time |
| 34 | |
| 35 | EXCLUDED_POD_NAMES = [ |
| 36 | "kube", "coredns", "kind", "grafana", |
| 37 | "prometheus", "tiller", "control-plane", |
Matteo Scandolo | 984cf7b | 2020-07-29 13:54:20 -0700 | [diff] [blame^] | 38 | "calico", "nginx", "registry", "local-path" |
Matteo Scandolo | 3ed8987 | 2020-07-15 17:01:02 -0700 | [diff] [blame] | 39 | ] |
| 40 | |
| 41 | DATE_FORMATTER_FN = mdates.DateFormatter('%Y-%m-%d %H:%M:%S') |
| 42 | |
| 43 | |
| 44 | def main(address, out_folder, since): |
| 45 | """ |
| 46 | Query Prometheus and generate .pdf files for CPU and Memory consumption for each POD |
| 47 | :param address: string The address of the Prometheus instance to query |
| 48 | :param out_folder: string The output folder (where to save the .pdf files) |
| 49 | :param since: int When to start collection data (minutes in the past) |
| 50 | :return: void |
| 51 | """ |
| 52 | time_delta = int(since) * 60 |
| 53 | container_mem_query = "container_memory_usage_bytes{image!=''}[%sm]" % since |
| 54 | container_cpu_query = "rate(container_cpu_user_seconds_total{image!=''}[%sm]) * 100" % since |
| 55 | |
| 56 | now = time.time() |
| 57 | cpu_params = { |
| 58 | "query": container_cpu_query, |
| 59 | "start": now - time_delta, |
| 60 | "end": now, |
| 61 | "step": "30", |
| 62 | } |
| 63 | r = requests.get("http://%s/api/v1/query_range" % address, cpu_params) |
| 64 | print("Downloading CPU info from: %s" % r.url) |
| 65 | container_cpu = r.json()["data"]["result"] |
Matteo Scandolo | 984cf7b | 2020-07-29 13:54:20 -0700 | [diff] [blame^] | 66 | containers = remove_unwanted_containers(container_cpu) |
| 67 | plot_cpu_consumption(containers, output="%s/cpu.pdf" % out_folder) |
| 68 | data_to_csv(containers, output="%s/cpu.csv" % out_folder) |
Matteo Scandolo | 3ed8987 | 2020-07-15 17:01:02 -0700 | [diff] [blame] | 69 | |
| 70 | r = requests.get("http://%s/api/v1/query" % address, {"query": container_mem_query}) |
| 71 | print("Downloading Memory info from: %s" % r.url) |
| 72 | container_mem = r.json()["data"]["result"] |
Matteo Scandolo | 984cf7b | 2020-07-29 13:54:20 -0700 | [diff] [blame^] | 73 | containers = remove_unwanted_containers(container_mem) |
| 74 | plot_memory_consumption(containers, output="%s/memory.pdf" % out_folder) |
| 75 | data_to_csv(containers, output="%s/memory.csv" % out_folder) |
| 76 | |
| 77 | |
| 78 | def data_to_csv(containers, output=None): |
| 79 | csv_file = open(output, "w+") |
| 80 | csv_writer = csv.writer(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) |
| 81 | |
| 82 | # we assume all the containers have the same timestamps |
| 83 | dates = [datetime.fromtimestamp(x[0]) for x in containers[0]["values"]] |
| 84 | csv_writer.writerow([''] + dates) |
| 85 | |
| 86 | for c in containers: |
| 87 | name = c["metric"]["pod"] |
| 88 | data = c["values"] |
| 89 | |
| 90 | values = [float(x[1]) for x in data] |
| 91 | csv_writer.writerow([name] + values) |
Matteo Scandolo | 3ed8987 | 2020-07-15 17:01:02 -0700 | [diff] [blame] | 92 | |
| 93 | |
| 94 | def plot_cpu_consumption(containers, output=None): |
Matteo Scandolo | 3ed8987 | 2020-07-15 17:01:02 -0700 | [diff] [blame] | 95 | plt.figure('cpu') |
| 96 | fig, ax = plt.subplots() |
| 97 | ax.xaxis.set_major_formatter(DATE_FORMATTER_FN) |
| 98 | ax.xaxis_date() |
| 99 | fig.autofmt_xdate() |
| 100 | |
| 101 | plt.title("CPU Usage per POD") |
| 102 | plt.xlabel("Timestamp") |
| 103 | plt.ylabel("% used") |
| 104 | |
| 105 | for c in containers: |
Matteo Scandolo | 984cf7b | 2020-07-29 13:54:20 -0700 | [diff] [blame^] | 106 | name = c["metric"]["pod"] |
Matteo Scandolo | 3ed8987 | 2020-07-15 17:01:02 -0700 | [diff] [blame] | 107 | data = c["values"] |
| 108 | |
| 109 | dates = [datetime.fromtimestamp(x[0]) for x in data] |
| 110 | |
| 111 | values = [float(x[1]) for x in data] |
| 112 | |
| 113 | plt.plot(dates, values, label=name, lw=2, color=get_line_color(name)) |
| 114 | # plt.plot(dates[1:], get_diff(values), label=name, lw=2, color=get_line_color(name)) |
| 115 | |
| 116 | plt.legend(loc='upper left') |
| 117 | |
| 118 | fig = plt.gcf() |
| 119 | fig.set_size_inches(20, 11) |
| 120 | |
| 121 | plt.savefig(output) |
| 122 | |
| 123 | |
| 124 | def plot_memory_consumption(containers, output=None): |
| 125 | plt.figure("memory") |
| 126 | fig, ax = plt.subplots() |
| 127 | ax.xaxis.set_major_formatter(DATE_FORMATTER_FN) |
| 128 | ax.xaxis_date() |
| 129 | fig.autofmt_xdate() |
| 130 | plt.title("Memory Usage") |
| 131 | plt.xlabel("Timestamp") |
| 132 | plt.ylabel("MB") |
| 133 | |
| 134 | for c in containers: |
Matteo Scandolo | 984cf7b | 2020-07-29 13:54:20 -0700 | [diff] [blame^] | 135 | name = c["metric"]["pod"] |
Matteo Scandolo | 3ed8987 | 2020-07-15 17:01:02 -0700 | [diff] [blame] | 136 | data = c["values"] |
| 137 | |
| 138 | dates = [datetime.fromtimestamp(x[0]) for x in data] |
| 139 | values = [bytesto(float(x[1]), "m") for x in data] |
| 140 | |
| 141 | plt.plot(dates[1:], get_diff(values), label=name, lw=2, color=get_line_color(name)) |
| 142 | |
| 143 | plt.legend(loc='upper left') |
| 144 | |
| 145 | fig = plt.gcf() |
| 146 | fig.set_size_inches(20, 11) |
| 147 | |
| 148 | plt.savefig(output) |
| 149 | |
| 150 | |
| 151 | def remove_unwanted_containers(cpus): |
| 152 | res = [] |
Matteo Scandolo | 984cf7b | 2020-07-29 13:54:20 -0700 | [diff] [blame^] | 153 | missed = [] |
Matteo Scandolo | 3ed8987 | 2020-07-15 17:01:02 -0700 | [diff] [blame] | 154 | for c in cpus: |
Matteo Scandolo | 984cf7b | 2020-07-29 13:54:20 -0700 | [diff] [blame^] | 155 | if "pod" in c["metric"]: |
Matteo Scandolo | 3ed8987 | 2020-07-15 17:01:02 -0700 | [diff] [blame] | 156 | |
Matteo Scandolo | 984cf7b | 2020-07-29 13:54:20 -0700 | [diff] [blame^] | 157 | if c["metric"]["id"].startswith("kubepods", 1): |
| 158 | missed.append(c) |
| 159 | continue |
| 160 | |
| 161 | if "container" not in c["metric"]: |
| 162 | missed.append(c) |
| 163 | continue |
| 164 | |
| 165 | pod_name = c["metric"]["pod"] |
Matteo Scandolo | 3ed8987 | 2020-07-15 17:01:02 -0700 | [diff] [blame] | 166 | container_name = c["metric"]["name"] |
| 167 | |
| 168 | if any(x in pod_name for x in EXCLUDED_POD_NAMES): |
Matteo Scandolo | 984cf7b | 2020-07-29 13:54:20 -0700 | [diff] [blame^] | 169 | missed.append(c) |
Matteo Scandolo | 3ed8987 | 2020-07-15 17:01:02 -0700 | [diff] [blame] | 170 | continue |
| 171 | |
| 172 | if "k8s_POD" in container_name: |
| 173 | # this is the kubernetes POD controller, we don't care about it |
Matteo Scandolo | 984cf7b | 2020-07-29 13:54:20 -0700 | [diff] [blame^] | 174 | missed.append(c) |
Matteo Scandolo | 3ed8987 | 2020-07-15 17:01:02 -0700 | [diff] [blame] | 175 | continue |
| 176 | |
| 177 | # if "_0" not in container_name: |
| 178 | # # this is something with the ONOS chart that is weird (each POD is reported 3 times) |
| 179 | # continue |
| 180 | |
| 181 | res.append(c) |
| 182 | else: |
| 183 | continue |
Matteo Scandolo | 984cf7b | 2020-07-29 13:54:20 -0700 | [diff] [blame^] | 184 | # print("Found containers: %s" % [c["metric"]["pod"] for c in res]) |
| 185 | # print("Missed containers: %s" % [c["metric"]["pod"] for c in missed]) |
Matteo Scandolo | 3ed8987 | 2020-07-15 17:01:02 -0700 | [diff] [blame] | 186 | return res |
| 187 | |
| 188 | |
| 189 | def get_line_color(container_name): |
| 190 | colors = { |
| 191 | "bbsim0": "#884EA0", |
| 192 | "bbsim1": "#9B59B6", |
| 193 | "bbsim-sadis-server": "#D2B4DE", |
| 194 | "onos-atomix-0": "#85C1E9", |
| 195 | "onos-atomix-1": "#7FB3D5", |
| 196 | "onos-atomix-2": "#3498DB", |
| 197 | "onos-onos-classic-0": "#1A5276", |
| 198 | "onos-onos-classic-1": "#1B4F72", |
| 199 | "onos-onos-classic-2": "#154360", |
| 200 | "etcd-0": "#7D6608", |
| 201 | "etcd-1": "#9A7D0A", |
| 202 | "etcd-2": "#B7950B", |
| 203 | "open-olt-voltha-adapter-openolt": "#7E5109", |
| 204 | "open-onu-voltha-adapter-openonu-0": "#6E2C00", |
| 205 | "open-onu-voltha-adapter-openonu-1": "#873600", |
| 206 | "open-onu-voltha-adapter-openonu-2": "#A04000", |
| 207 | "open-onu-voltha-adapter-openonu-3": "#BA4A00", |
| 208 | "open-onu-voltha-adapter-openonu-4": "#D35400", |
| 209 | "open-onu-voltha-adapter-openonu-5": "#D35400", |
| 210 | "open-onu-voltha-adapter-openonu-6": "#E59866", |
| 211 | "open-onu-voltha-adapter-openonu-7": "#EDBB99", |
| 212 | "kafka-0": "#4D5656", |
| 213 | "kafka-1": "#5F6A6A", |
| 214 | "kafka-2": "#717D7E", |
| 215 | "kafka-zookeeper-0": "#839192", |
| 216 | "kafka-zookeeper-1": "#95A5A6", |
| 217 | "kafka-zookeeper-2": "#717D7E", |
| 218 | "radius": "#82E0AA", |
| 219 | "voltha-voltha-ofagent": "#641E16", |
| 220 | "voltha-voltha-rw-core": "#7B241C", |
| 221 | } |
| 222 | |
| 223 | if container_name in colors: |
| 224 | return colors[container_name] |
| 225 | elif "openolt" in container_name: |
| 226 | return colors["open-olt-voltha-adapter-openolt"] |
| 227 | elif "ofagent" in container_name: |
| 228 | return colors["voltha-voltha-ofagent"] |
| 229 | elif "rw-core" in container_name: |
| 230 | return colors["voltha-voltha-rw-core"] |
| 231 | elif "bbsim0" in container_name: |
| 232 | return colors["bbsim0"] |
| 233 | elif "bbsim1" in container_name: |
| 234 | return colors["bbsim1"] |
| 235 | elif "bbsim-sadis-server" in container_name: |
| 236 | return colors["bbsim-sadis-server"] |
| 237 | elif "radius" in container_name: |
| 238 | return colors["radius"] |
| 239 | else: |
| 240 | return "black" |
| 241 | |
| 242 | |
| 243 | def get_diff(data): |
| 244 | return [x - data[i - 1] for i, x in enumerate(data)][1:] |
| 245 | |
| 246 | |
| 247 | def bytesto(b, to, bsize=1024): |
| 248 | """convert bytes to megabytes, etc. |
| 249 | sample code: |
| 250 | print('mb= ' + str(bytesto(314575262000000, 'm'))) |
| 251 | sample output: |
| 252 | mb= 300002347.946 |
| 253 | """ |
| 254 | |
| 255 | a = {'k': 1, 'm': 2, 'g': 3, 't': 4, 'p': 5, 'e': 6} |
| 256 | r = float(b) |
| 257 | for i in range(a[to]): |
| 258 | r = r / bsize |
| 259 | |
| 260 | return r |
| 261 | |
| 262 | |
| 263 | if __name__ == "__main__": |
| 264 | parser = argparse.ArgumentParser(prog="sizing") |
| 265 | parser.add_argument("-a", "--address", help="The address of the Prometheus instance we're targeting", |
| 266 | default="127.0.0.1:31301") |
| 267 | parser.add_argument("-o", "--output", help="Where to output the generated files", |
| 268 | default="plots") |
| 269 | parser.add_argument("-s", "--since", help="When to start sampling the data (in minutes before now)", |
| 270 | default=10) |
| 271 | |
| 272 | args = parser.parse_args() |
| 273 | main(args.address, args.output, args.since) |