blob: f53d4177151b1904cc2cd7ec13c9aed251757b11 [file] [log] [blame]
Srikanth Vavilapalli1ea11fc2015-12-14 00:52:57 -05001import socket
2import requests
3import urllib2
4import json
5import msgpack
6import collections
7import time, thread, threading
8
Srikanth Vavilapallia6de6a32015-12-16 14:55:30 -05009from flask import request, Request, jsonify
10from flask import Flask
11from flask import make_response
12app = Flask(__name__)
13
Srikanth Vavilapalli1ea11fc2015-12-14 00:52:57 -050014projects_map = {}
Srikanth Vavilapalli73afe292015-12-14 17:35:04 -050015xos_tenant_info_map = {}
16xos_instances_info_map = {}
Srikanth Vavilapalli1ea11fc2015-12-14 00:52:57 -050017
18UDP_IP = "0.0.0.0"
19UDP_PORT = 12346
20
Srikanth Vavilapallia6de6a32015-12-16 14:55:30 -050021@app.route('/autoscaledata',methods=['GET'])
22def autoscaledata():
teone3aa413d2015-12-16 18:28:14 -050023 response = app.make_response(json.dumps(projects_map.values()))
Srikanth Vavilapallia6de6a32015-12-16 14:55:30 -050024 response.mimetype="application/json"
25 return response
26
Srikanth Vavilapalli1ea11fc2015-12-14 00:52:57 -050027def acquire_xos_monitoring_channel():
28 url = "http://ctl:9999/xoslib/monitoringchannel/"
29 admin_auth=("padmin@vicci.org", "letmein") # use your XOS username and password
30 monitoring_channels = requests.get(url, auth=admin_auth).json()
31 ceilometer_url = None
32 if not monitoring_channels:
33 print 'SRIKANTH: No monitoring channels for this user...'
34 return None
35 else:
36 monitoring_channel = monitoring_channels[0]
37 while not monitoring_channel['ceilometer_url']:
38 print 'SRIKANTH: Waiting for monitoring channel create'
39 sleep(0.5)
40 monitoring_channel = requests.get(url, auth=admin_auth).json()[0]
41 #TODO: Wait until URL is completely UP
42 while True:
43 print 'SRIKANTH: Waiting for ceilometer proxy URL %s is available' % monitoring_channel['ceilometer_url']
44 try:
45 response = urllib2.urlopen(monitoring_channel['ceilometer_url'],timeout=1)
46 break
47 except urllib2.HTTPError, e:
48 print 'SRIKANTH: HTTP error %s' % e.reason
49 break
50 except urllib2.URLError, e:
51 print 'SRIKANTH: URL error %(reason)s' % e.reason
52 pass
53 return monitoring_channel
54
55def print_samples():
56 print ""
57 print ""
58 for project in projects_map.keys():
teone3aa413d2015-12-16 18:28:14 -050059 print "service=%s slice=%s, alarm_state=%s" % (projects_map[project]['service'], projects_map[project]['slice'] if projects_map[project]['slice'] else project, projects_map[project]['alarm'])
Srikanth Vavilapalli1ea11fc2015-12-14 00:52:57 -050060 for resource in projects_map[project]['resources'].keys():
Srikanth Vavilapalli73afe292015-12-14 17:35:04 -050061 print "resource=%s" % (projects_map[project]['resources'][resource]['xos_instance_info']['instance_name'] if projects_map[project]['resources'][resource]['xos_instance_info'] else resource)
62 for i in projects_map[project]['resources'][resource]['queue']:
Srikanth Vavilapalli1ea11fc2015-12-14 00:52:57 -050063 print " time=%s val=%s" % ( i['timestamp'],i['counter_volume'])
64
65def periodic_print():
66 print_samples()
67 #Print every 1minute
68 threading.Timer(60, periodic_print).start()
69
70
71CPU_UPPER_THRESHOLD = 80 #80%
72CPU_LOWER_THRESHOLD = 30 #30%
73CPU_THRESHOLD_REPEAT = 3
74INITIAL_STATE = 'normal_config'
75SCALE_UP_EVALUATION = 'scale_up_eval'
76SCALE_DOWN_EVALUATION = 'scale_down_eval'
77SCALE_UP_ALARM = 'scale_up'
78SCALE_DOWN_ALARM = 'scale_down'
79
Srikanth Vavilapalli73afe292015-12-14 17:35:04 -050080def loadAllXosTenantInfo():
81 print "SRIKANTH: Loading all XOS tenant info"
Srikanth Vavilapalli1ea11fc2015-12-14 00:52:57 -050082 url = "http://ctl:9999/xos/controllerslices/"
83 admin_auth=("padmin@vicci.org", "letmein") # use your XOS username and password
84 controller_slices = requests.get(url, auth=admin_auth).json()
85 for cslice in controller_slices:
Srikanth Vavilapalli73afe292015-12-14 17:35:04 -050086 slice = requests.get(cslice['slice'], auth=admin_auth).json()
87 slice_name = slice['humanReadableName']
88 if slice['service']:
Srikanth Vavilapalli1ea11fc2015-12-14 00:52:57 -050089 service = requests.get(slice['service'], auth=admin_auth).json()
90 service_name = service['humanReadableName']
Srikanth Vavilapalli73afe292015-12-14 17:35:04 -050091 else:
92 service_name = None
93 xos_tenant_info_map[cslice['tenant_id']] = {'service':service_name, 'slice':slice_name}
94 print "SRIKANTH: Project: %s Service:%s Slice:%s" % (cslice['tenant_id'],service_name,slice_name)
95
96def loadAllXosInstanceInfo():
97 print "SRIKANTH: Loading all XOS instance info"
teone3aa413d2015-12-16 18:28:14 -050098 url = "http://ctl:9999/xos/instances/"
Srikanth Vavilapalli73afe292015-12-14 17:35:04 -050099 admin_auth=("padmin@vicci.org", "letmein") # use your XOS username and password
100 xos_instances = requests.get(url, auth=admin_auth).json()
101 for instance in xos_instances:
102 xos_instances_info_map[instance['instance_uuid']] = {'instance_name':instance['instance_name']}
103
104def getXosTenantInfo(project):
105 xos_tenant_info = xos_tenant_info_map.get(project, None)
106 if xos_tenant_info:
107 return xos_tenant_info
108 else:
109 loadAllXosTenantInfo()
110 xos_tenant_info = xos_tenant_info_map.get(project, None)
111 if not xos_tenant_info:
112 print "SRIKANTH: Project %s has no associated XOS slice" % project
113 return xos_tenant_info
114
115def getXosInstanceInfo(resource):
116 xos_instance_info = xos_instances_info_map.get(resource, None)
117 if xos_instance_info:
118 return xos_instance_info
119 else:
120 loadAllXosInstanceInfo()
121 xos_instance_info = xos_instances_info_map.get(resource, None)
122 if not xos_instance_info:
123 print "SRIKANTH: Resource %s has no associated XOS instance" % project
124 return xos_instance_info
Srikanth Vavilapalli1ea11fc2015-12-14 00:52:57 -0500125
126def handle_adjust_scale(project, adjust):
127 if (adjust != 'up') and (adjust != 'down'):
128 print "SRIKANTH: Invalid adjust value %s " % adjust
129 return
130 current_instances = len(projects_map[project]['resources'].keys())
131 if (current_instances <=1 and adjust == 'down'):
132 print "SRIKANTH: %s is running with already minimum instances and can not scale down further " % project
133 return
134 if (current_instances >=2 and adjust == 'up'):
135 print "SRIKANTH: %s is running with already maximum instances and can not scale up further " % project
136 return
Srikanth Vavilapalli73afe292015-12-14 17:35:04 -0500137 #xos_tenant = getXosTenantInfo(project)
teone3aa413d2015-12-16 18:28:14 -0500138 xos_service = projects_map[project]['service']
139 xos_slice = projects_map[project]['slice']
Srikanth Vavilapalli1ea11fc2015-12-14 00:52:57 -0500140 if not xos_service or not xos_slice:
141 print "SRIKANTH: Can not handle adjust_scale for Project %s because not associated with any service or slice" % project
142 return
143 print "SRIKANTH: SCALE %s for Project %s, Slice=%s, Service=%s from current=%d to new=%d" % (adjust, project, xos_slice, xos_service, current_instances, current_instances+1 if (adjust=='up') else current_instances-1)
144 query_params = {'service':xos_service, 'slice_hint':xos_slice, 'scale':current_instances+1 if (adjust=='up') else current_instances-1}
145 url = "http://ctl:9999/xoslib/serviceadjustscale/"
146 admin_auth=("padmin@vicci.org", "letmein") # use your XOS username and password
147 response = requests.get(url, params=query_params, auth=admin_auth).json()
148 print "SRIKANTH: XOS adjust_scale response: %s" % response
149
150def periodic_cpu_threshold_evaluator():
151 for project in projects_map.keys():
Srikanth Vavilapalli73afe292015-12-14 17:35:04 -0500152 aggregate_cpu_util = sum([resource['queue'][-1]['counter_volume'] \
153 for resource in projects_map[project]['resources'].values()]) \
Srikanth Vavilapalli1ea11fc2015-12-14 00:52:57 -0500154 /len(projects_map[project]['resources'].keys())
155
156 if (projects_map[project]['alarm'] == INITIAL_STATE or
157 projects_map[project]['alarm'] == SCALE_UP_ALARM or
158 projects_map[project]['alarm'] == SCALE_DOWN_ALARM):
159 if aggregate_cpu_util > CPU_UPPER_THRESHOLD:
160 projects_map[project]['uthreadshold_count'] = 1
161 projects_map[project]['alarm'] = SCALE_UP_EVALUATION
162 if projects_map[project]['uthreadshold_count'] >= CPU_THRESHOLD_REPEAT:
163 projects_map[project]['alarm'] = SCALE_UP_ALARM
164 handle_adjust_scale(project, 'up')
165 elif aggregate_cpu_util < CPU_LOWER_THRESHOLD:
166 projects_map[project]['lthreadshold_count'] = 1
167 projects_map[project]['alarm'] = SCALE_DOWN_EVALUATION
168 if projects_map[project]['lthreadshold_count'] >= CPU_THRESHOLD_REPEAT:
169 projects_map[project]['alarm'] = SCALE_DOWN_ALARM
170 handle_adjust_scale(project, 'down')
171 else:
172 projects_map[project]['uthreadshold_count'] = 0
173 projects_map[project]['lthreadshold_count'] = 0
174 projects_map[project]['alarm'] = INITIAL_STATE
175 elif projects_map[project]['alarm'] == SCALE_UP_EVALUATION:
176 if aggregate_cpu_util > CPU_UPPER_THRESHOLD:
177 projects_map[project]['uthreadshold_count'] += 1
178 if projects_map[project]['uthreadshold_count'] >= CPU_THRESHOLD_REPEAT:
179 projects_map[project]['alarm'] = SCALE_UP_ALARM
180 handle_adjust_scale(project, 'up')
181 elif aggregate_cpu_util < CPU_LOWER_THRESHOLD:
182 projects_map[project]['lthreadshold_count'] += 1
183 projects_map[project]['alarm'] = SCALE_DOWN_EVALUATION
184 else:
185 projects_map[project]['uthreadshold_count'] = 0
186 projects_map[project]['alarm'] = INITIAL_STATE
187 elif projects_map[project]['alarm'] == SCALE_DOWN_EVALUATION:
188 if aggregate_cpu_util < CPU_LOWER_THRESHOLD:
189 projects_map[project]['lthreadshold_count'] += 1
190 if projects_map[project]['lthreadshold_count'] >= CPU_THRESHOLD_REPEAT:
191 projects_map[project]['alarm'] = SCALE_DOWN_ALARM
192 handle_adjust_scale(project, 'down')
193 elif aggregate_cpu_util > CPU_UPPER_THRESHOLD:
194 projects_map[project]['uthreadshold_count'] += 1
195 projects_map[project]['alarm'] = SCALE_UP_EVALUATION
196 else:
197 projects_map[project]['lthreadshold_count'] = 0
198 projects_map[project]['alarm'] = INITIAL_STATE
199 threading.Timer(60, periodic_cpu_threshold_evaluator).start()
200
201def read_notification_from_ceilometer(host,port):
202 udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
203 udp.bind((host, port))
204
205 while True:
206 data, source = udp.recvfrom(64000)
207 try:
208 sample = msgpack.loads(data, encoding='utf-8')
209 if sample['counter_name'] != 'cpu_util':
210 continue
211 if sample['project_id'] not in projects_map.keys():
212 projects_map[sample['project_id']] = {}
teone3aa413d2015-12-16 18:28:14 -0500213 xosTenantInfo = getXosTenantInfo(sample['project_id'])
214 projects_map[sample['project_id']]['project_id'] = sample['project_id']
215 projects_map[sample['project_id']]['slice'] = xosTenantInfo['slice']
216 projects_map[sample['project_id']]['service'] = xosTenantInfo['service']
Srikanth Vavilapalli1ea11fc2015-12-14 00:52:57 -0500217 projects_map[sample['project_id']]['resources'] = {}
218 projects_map[sample['project_id']]['uthreadshold_count'] = 0
219 projects_map[sample['project_id']]['lthreadshold_count'] = 0
220 projects_map[sample['project_id']]['alarm'] = INITIAL_STATE
221 resource_map = projects_map[sample['project_id']]['resources']
222 if sample['resource_id'] not in resource_map.keys():
Srikanth Vavilapalli73afe292015-12-14 17:35:04 -0500223 resource_map[sample['resource_id']] = {}
224 resource_map[sample['resource_id']]['xos_instance_info'] = getXosInstanceInfo(sample['resource_id'])
Srikanth Vavilapallia6de6a32015-12-16 14:55:30 -0500225 resource_map[sample['resource_id']]['queue'] = []
226 samples_queue = resource_map[sample['resource_id']]['queue']
Srikanth Vavilapalli1ea11fc2015-12-14 00:52:57 -0500227 sample = {'counter_name':sample['counter_name'],
228 'project_id':sample['project_id'],
229 'resource_id':sample['resource_id'],
230 'timestamp':sample['timestamp'],
231 'counter_unit':sample['counter_unit'],
232 'counter_volume':sample['counter_volume']}
Srikanth Vavilapallia6de6a32015-12-16 14:55:30 -0500233 deque = collections.deque(samples_queue, maxlen=10)
234 deque.append(sample)
235 resource_map[sample['resource_id']]['queue'] = list(deque)
Srikanth Vavilapalli1ea11fc2015-12-14 00:52:57 -0500236 except Exception as e:
237 print e
238
Srikanth Vavilapallia6de6a32015-12-16 14:55:30 -0500239def setup_webserver():
240 try:
241 #config = ConfigParser.ConfigParser()
242 #config.read('pub_sub.conf')
243 #webserver_host = config.get('WEB_SERVER','webserver_host')
244 #webserver_port = int (config.get('WEB_SERVER','webserver_port'))
245 #client_host = config.get('CLIENT','client_host')
246 #client_port = int (config.get('CLIENT','client_port'))
247
248 #log_level = config.get('LOGGING','level')
249 #log_file = config.get('LOGGING','filename')
250
251 #level = LEVELS.get(log_level, logging.NOTSET)
252 #logging.basicConfig(filename=log_file,format='%(asctime)s %(levelname)s %(message)s',\
253 # datefmt=_DEFAULT_LOG_DATE_FORMAT,level=level)
254 webserver_host = '0.0.0.0'
255 webserver_port = 9991
256
257 except Exception as e:
258 print("* Error in config file:",e.__str__())
259 logging.error("* Error in confing file:%s",e.__str__())
260 else:
261 app.run(host=webserver_host,port=webserver_port,debug=True, use_reloader=False)
262
263
Srikanth Vavilapalli1ea11fc2015-12-14 00:52:57 -0500264def main():
265 monitoring_channel = acquire_xos_monitoring_channel()
266 if not monitoring_channel:
267 print 'SRIKANTH: XOS monitoring_channel is not created... Create it before using this app'
268 return
Srikanth Vavilapalli73afe292015-12-14 17:35:04 -0500269 loadAllXosTenantInfo()
270 loadAllXosInstanceInfo()
Srikanth Vavilapalli1ea11fc2015-12-14 00:52:57 -0500271 thread.start_new(read_notification_from_ceilometer,(UDP_IP,UDP_PORT,))
272 ceilometer_url = monitoring_channel['ceilometer_url']
273 subscribe_data = {"sub_info":"cpu_util","app_id":"xos_auto_scale","target":"udp://10.11.10.1:12346"}
274 subscribe_url = ceilometer_url + 'v2/subscribe'
275 response = requests.post(subscribe_url, data=json.dumps(subscribe_data))
276 print 'SRIKANTH: Ceilometer Subscription status:%s' % response.text
277 #TODO: Fix the typo in 'sucess'
278 if (not 'sucess' in response.text) and (not 'already exists' in response.text):
279 print 'SRIKANTH: Ceilometer Subscription unsuccessful...Exiting'
280 return
281 periodic_cpu_threshold_evaluator()
282 periodic_print()
Srikanth Vavilapallia6de6a32015-12-16 14:55:30 -0500283 setup_webserver()
Srikanth Vavilapalli1ea11fc2015-12-14 00:52:57 -0500284
285if __name__ == "__main__":
286 main()