CORD-782: Fixes in auto-scale application to be supported on top of CORD monitoring service
Change-Id: Iafa30205a81ad7a555180dc370af537ba4fd7719
diff --git a/auto-scale/README.md b/auto-scale/README.md
index 90e6c2c..7fae9ac 100644
--- a/auto-scale/README.md
+++ b/auto-scale/README.md
@@ -21,10 +21,12 @@
Ensure that XOS REST IP address is accessible from where this application is installed and then run:
-`python xos_auto_scaling_app.py --xos-ip=<xos-ip> --kafka-ip=<monitoringservice-kafka-ip>`
+`python xos_auto_scaling_app.py --xos-ip=<xos-ip> --kafka-ip=<monitoringservice-kafka-ip> --supported-services=<comma seperated service names>`
This command will start the autoscaling application and start REST server on 9991 port.
+NOTE: If no services are provided as --supported-services argument, then this application will not perform aut-scaling for any XOS services
+
## To verify the autoscaling application:
1) Login to cloudlab compute nodes <br/>
2) On each compute node, open /etc/ceilometer/pipeline.yaml file<br/>
diff --git a/auto-scale/xos_auto_scaling_app.py b/auto-scale/xos_auto_scaling_app.py
index 9ec0103..59ee926 100644
--- a/auto-scale/xos_auto_scaling_app.py
+++ b/auto-scale/xos_auto_scaling_app.py
@@ -76,10 +76,16 @@
return monitoring_channel
def print_samples():
+ global supported_services
+
print ""
print ""
for project in projects_map.keys():
- print "service=%s slice=%s, alarm_state=%s" % (projects_map[project]['service'], projects_map[project]['slice'] if projects_map[project]['slice'] else project, projects_map[project]['alarm'])
+ xos_service = projects_map[project]['service']
+ if (not xos_service) or (xos_service not in supported_services):
+ print "Skipped scale evaluation for project=%s and service =%s" % (project, xos_service)
+ continue
+ print "service=%s slice=%s, alarm_state=%s lcount=%s ucount=%s" % (projects_map[project]['service'], projects_map[project]['slice'] if projects_map[project]['slice'] else project, projects_map[project]['alarm'], projects_map[project]['lthreadshold_count'], projects_map[project]['uthreadshold_count'])
for resource in projects_map[project]['resources'].keys():
print "resource=%s" % (projects_map[project]['resources'][resource]['xos_instance_info']['instance_name'] if projects_map[project]['resources'][resource]['xos_instance_info'] else resource)
for i in projects_map[project]['resources'][resource]['queue']:
@@ -87,7 +93,7 @@
def periodic_print():
print_samples()
- #Print every 1minute
+ #Print every 20 seconds
threading.Timer(20, periodic_print).start()
@@ -147,39 +153,48 @@
loadAllXosInstanceInfo()
xos_instance_info = xos_instances_info_map.get(resource, None)
if not xos_instance_info:
- print "Resource %s has no associated XOS instance" % project
+ print "Resource %s has no associated XOS instance" % resource
return xos_instance_info
def handle_adjust_scale(project, adjust):
- global xos_ip
+ global xos_ip, supported_services
if (adjust != 'up') and (adjust != 'down'):
print "SRIKANTH: Invalid adjust value %s " % adjust
return
- current_instances = len(projects_map[project]['resources'].keys())
- if (current_instances <=1 and adjust == 'down'):
- print "%s is running with already minimum instances and can not scale down further " % project
- return
- if (current_instances >=2 and adjust == 'up'):
- print "%s is running with already maximum instances and can not scale up further " % project
- return
- #xos_tenant = getXosTenantInfo(project)
xos_service = projects_map[project]['service']
xos_slice = projects_map[project]['slice']
if not xos_service or not xos_slice:
print "Can not handle adjust_scale for Project %s because not associated with any service or slice" % project
return
+ if (xos_service not in supported_services):
+ print "SRIKANTH: Scaling is not supported for this service %s...Ignoring adjust_scale request" % xos_service
+ return
+ current_instances = len(projects_map[project]['resources'].keys())
+ if (current_instances <=1 and adjust == 'down'):
+ print "%s is running with already minimum instances and can not scale down further " % xos_service
+ return
+ if (current_instances >=2 and adjust == 'up'):
+ print "%s is running with already maximum instances and can not scale up further " % xos_service
+ return
+ #xos_tenant = getXosTenantInfo(project)
print "SCALE %s for Project %s, Slice=%s, Service=%s from current=%d to new=%d" % (adjust, project, xos_slice, xos_service, current_instances, current_instances+1 if (adjust=='up') else current_instances-1)
query_params = {'service':xos_service, 'slice_hint':xos_slice, 'scale':current_instances+1 if (adjust=='up') else current_instances-1}
- url = "http://" + xos_ip + ":" + str(xos_port) + "/xoslib/serviceadjustscale/"
+ url = "http://" + xos_ip + ":" + str(xos_port) + "/api/tenant/monitoring/dashboard/serviceadjustscale/"
admin_auth=("padmin@vicci.org", "letmein") # use your XOS username and password
response = requests.get(url, params=query_params, auth=admin_auth).json()
print "SRIKANTH: XOS adjust_scale response: %s" % response
def periodic_cpu_threshold_evaluator():
+ global supported_services
+
for project in projects_map.keys():
+ xos_service = projects_map[project]['service']
+ if (not xos_service) or (xos_service not in supported_services):
+ #print "Skipping scale evaluation for project=%s and service =%s" % (project, xos_service)
+ continue
aggregate_cpu_util = sum([resource['queue'][-1]['counter_volume'] \
- for resource in projects_map[project]['resources'].values()]) \
+ for resource in projects_map[project]['resources'].values() if 'queue' in resource.keys()]) \
/len(projects_map[project]['resources'].keys())
if (projects_map[project]['alarm'] == INITIAL_STATE or
@@ -286,8 +301,8 @@
projects_map[sample['project_id']] = {}
xosTenantInfo = getXosTenantInfo(sample['project_id'])
projects_map[sample['project_id']]['project_id'] = sample['project_id']
- projects_map[sample['project_id']]['slice'] = xosTenantInfo['slice']
- projects_map[sample['project_id']]['service'] = xosTenantInfo['service']
+ projects_map[sample['project_id']]['slice'] = (xosTenantInfo['slice'] if xosTenantInfo else None)
+ projects_map[sample['project_id']]['service'] = (xosTenantInfo['service'] if xosTenantInfo else None)
projects_map[sample['project_id']]['resources'] = {}
projects_map[sample['project_id']]['uthreadshold_count'] = 0
projects_map[sample['project_id']]['lthreadshold_count'] = 0
@@ -333,12 +348,12 @@
app.run(host=webserver_host,port=webserver_port,debug=True, use_reloader=False)
def parse_args(argv):
- global xos_ip, kafka_ip
+ global xos_ip, kafka_ip, supported_services
try:
- opts, args = getopt.getopt(argv,"k:x:",["kafka-ip=","xos-ip="])
+ opts, args = getopt.getopt(argv,"k:s:x:",["kafka-ip=","supported-services=","xos-ip="])
except getopt.GetoptError:
- print 'xos_auto_scaling_app.py --xos-ip=<IP> --kafka-ip=<>'
+ print 'xos_auto_scaling_app.py --xos-ip=<IP> --kafka-ip=<> [--supported-services=<comma seperated service name>]'
sys.exit(2)
for opt, arg in opts:
@@ -346,10 +361,13 @@
xos_ip = arg
elif opt in ("--kafka-ip"):
kafka_ip = arg
+ elif opt in ("--supported-services"):
+ supported_services = arg.split(',')
if not xos_ip or not kafka_ip:
- print 'xos_auto_scaling_app.py --xos-ip=<IP> --kafka-ip=<>'
+ print 'xos_auto_scaling_app.py --xos-ip=<IP> --kafka-ip=<> [--supported-services=<comma seperated service name>]'
sys.exit(2)
+ print 'xos-ip=%s, kafka-ip=%s, supported-services=%s' % (xos_ip, kafka_ip, supported_services)
def main(argv):