Adding backedn_status info to the monitoring service and tenant REST APIs
Adding e2e tests for monitoring service
Change-Id: I9f60f380318a84e2b13535f638920274c64ac30d
diff --git a/xos/api/service/monitoring/monitoringservice.py b/xos/api/service/monitoring/monitoringservice.py
new file mode 100644
index 0000000..37987cb
--- /dev/null
+++ b/xos/api/service/monitoring/monitoringservice.py
@@ -0,0 +1,79 @@
+from rest_framework.decorators import api_view
+from rest_framework.response import Response
+from rest_framework.reverse import reverse
+from rest_framework import serializers
+from rest_framework import generics
+from rest_framework import viewsets
+from rest_framework import status
+from rest_framework.decorators import detail_route, list_route
+from rest_framework.views import APIView
+from core.models import *
+from django.forms import widgets
+from django.conf.urls import patterns, url
+from api.xosapi_helpers import PlusModelSerializer, XOSViewSet, ReadOnlyField
+from django.shortcuts import get_object_or_404
+from xos.apibase import XOSListCreateAPIView, XOSRetrieveUpdateDestroyAPIView, XOSPermissionDenied
+from xos.exceptions import *
+import json
+import subprocess
+from django.views.decorators.csrf import ensure_csrf_cookie
+from services.monitoring.models import MonitoringChannel, CeilometerService
+
+class CeilometerServiceForApi(CeilometerService):
+ class Meta:
+ proxy = True
+ app_label = "ceilometer"
+
+ @property
+ def related(self):
+ related = {}
+ if self.creator:
+ related["creator"] = self.creator.username
+ instance = self.get_instance()
+ if instance:
+ related["instance_id"] = instance.id
+ related["instance_name"] = instance.name
+ if instance.node:
+ related["compute_node_name"] = instance.node.name
+ return related
+
+
+class CeilometerServiceSerializer(PlusModelSerializer):
+ id = ReadOnlyField()
+ backend_status = ReadOnlyField()
+ humanReadableName = serializers.SerializerMethodField("getHumanReadableName")
+ ceilometer_pub_sub_url = ReadOnlyField()
+ related = serializers.DictField(required=False)
+
+ class Meta:
+ model = CeilometerServiceForApi
+ fields = ('humanReadableName',
+ 'id',
+ 'backend_status',
+ 'ceilometer_pub_sub_url',
+ 'related')
+
+ def getHumanReadableName(self, obj):
+ return obj.__unicode__()
+
+# @ensure_csrf_cookie
+class CeilometerServiceViewSet(XOSViewSet):
+ base_name = "monitoringservice"
+ method_name = None # use the api endpoint /api/service/monitoring/
+ method_kind = "viewset"
+ queryset = CeilometerServiceForApi.get_service_objects().select_related().all()
+ serializer_class = CeilometerServiceSerializer
+
+ @classmethod
+ def get_urlpatterns(self, api_path="^"):
+ patterns = super(CeilometerServiceViewSet, self).get_urlpatterns(api_path=api_path)
+
+ return patterns
+
+ def list(self, request):
+ object_list = self.filter_queryset(self.get_queryset())
+
+ serializer = self.get_serializer(object_list, many=True)
+
+ return Response(serializer.data)
+
diff --git a/xos/api/tenant/monitoring/monitoringchannel.py b/xos/api/tenant/monitoring/monitoringchannel.py
index 8385c40..6324745 100644
--- a/xos/api/tenant/monitoring/monitoringchannel.py
+++ b/xos/api/tenant/monitoring/monitoringchannel.py
@@ -36,6 +36,7 @@
class MonitoringChannelSerializer(PlusModelSerializer):
id = ReadOnlyField()
+ backend_status = ReadOnlyField()
service_specific_attribute = ReadOnlyField()
ceilometer_url = ReadOnlyField()
tenant_list_str = ReadOnlyField()
@@ -50,7 +51,7 @@
class Meta:
model = MonitoringChannelForAPI
- fields = ('humanReadableName', 'id', 'provider_service', 'service_specific_attribute', 'ceilometer_url', 'tenant_list_str', 'related' )
+ fields = ('humanReadableName', 'id', 'backend_status', 'provider_service', 'service_specific_attribute', 'ceilometer_url', 'tenant_list_str', 'related' )
def getHumanReadableName(self, obj):
return obj.__unicode__()
diff --git a/xos/monitoring-onboard.yaml b/xos/monitoring-onboard.yaml
index 21a6174..ad7352b 100644
--- a/xos/monitoring-onboard.yaml
+++ b/xos/monitoring-onboard.yaml
@@ -20,6 +20,7 @@
synchronizer_run: monitoring_synchronizer.py
tosca_custom_types: monitoring_tosca_types.yaml
tosca_resource: tosca/resources/ceilometerservice.py, tosca/resources/ceilometertenant.py, tosca/resources/sflowservice.py, tosca/resources/openstackmonitoringpublisher.py, tosca/resources/onosmonitoringpublisher.py, tosca/resources/userservicemonitoringpublisher.py, tosca/resources/inframonitoringagentinfo.py, tosca/resources/monitoringcollectorplugininfo.py
+ rest_service: subdirectory:monitoring api/service/monitoring/monitoringservice.py
rest_tenant: subdirectory:monitoring api/tenant/monitoring/monitoringchannel.py, subdirectory:monitoring/dashboard api/tenant/monitoring/dashboard/ceilometerdashboard.py
private_key: file:///opt/xos/key_import/monitoringservice_rsa
public_key: file:///opt/xos/key_import/monitoringservice_rsa.pub
diff --git a/xos/test/monitoring_e2e_test.yaml b/xos/test/monitoring_e2e_test.yaml
new file mode 100644
index 0000000..0cdb4d1
--- /dev/null
+++ b/xos/test/monitoring_e2e_test.yaml
@@ -0,0 +1,13 @@
+---
+- hosts: localhost
+
+ vars:
+ auth:
+ user: padmin@vicci.org
+ pass: letmein
+
+ roles:
+ - {role: common}
+ - {role: test-initial, when: "{{test}}==1"}
+ - {role: test-inframonitoring, when: "{{test}}==2"}
+ - {role: test-vsgmonitoring, when: "{{test}}==3"}
diff --git a/xos/test/monitoringservice_test.py b/xos/test/monitoringservice_test.py
new file mode 100644
index 0000000..73e33c0
--- /dev/null
+++ b/xos/test/monitoringservice_test.py
@@ -0,0 +1,196 @@
+import urllib2
+import requests
+import time
+import json
+import sys
+import getopt
+
+monitoring_channel = None
+
+def acquire_xos_monitoring_channel():
+ admin_auth=("padmin@vicci.org", "letmein")
+ monitoring_channel = None
+ ceilometer_service = None
+ start_time = time.time()
+ ceilometerservice_wait_start_time=start_time
+ monitoringchannel_wait_start_time=start_time
+ cur_attempts = 1
+ print "Attempt %s" % cur_attempts
+ while True:
+ try:
+ if (not ceilometer_service):
+ url = "http://localhost:8888/api/core/services/"
+ services = requests.get(url, auth=admin_auth).json()
+ if not services:
+ print 'No services are found....weird....exiting'
+ return None
+ else:
+ for service in services:
+ if 'ceilometer' in service['name']:
+ ceilometer_service = service
+ break
+ if (not ceilometer_service):
+ print 'Waiting for ceilometer_service object to be created, elapsed-time=%s' % (time.time()-ceilometerservice_wait_start_time)
+ else:
+ if ("OK" not in ceilometer_service['backend_status']):
+ cur_status = "other"
+ if "Unreachable" in ceilometer_service['backend_status']:
+ cur_status = "Unreachable"
+ elif "defer" in ceilometer_service['backend_status']:
+ cur_status = "Deferred"
+ print 'Waiting for ceilometer_service object to be ready, current_status:%s elapsed-time:%s' % (cur_status, time.time()-ceilometerservice_wait_start_time)
+ ceilometer_service = None
+ else:
+ print 'ceilometer_service is ready, elapsed-time:%s' % (time.time()-ceilometerservice_wait_start_time)
+ else:
+ print 'ceilometer_service is ready'
+ except Exception, e:
+ print 'Waiting for ceilometer_service object to be created, elapsed-time=%s' % (time.time()-ceilometerservice_wait_start_time)
+
+ try:
+ if (not monitoring_channel):
+ url = "http://localhost:8888/api/tenant/monitoring/monitoringchannel/"
+ monitoring_channels = requests.get(url, auth=admin_auth).json()
+ if not monitoring_channels:
+ print 'Waiting for monitoring_channel object to be created, elapsed-time=%s' % (time.time()-monitoringchannel_wait_start_time)
+ else:
+ monitoring_channel = monitoring_channels[0]
+ url = "http://localhost:8888/api/core/tenants/"+str(monitoring_channel['id'])
+ monitoring_channel = None
+ monitoring_channel = requests.get(url, auth=admin_auth).json()
+ if (not monitoring_channel) or ("OK" not in monitoring_channel['backend_status']):
+ cur_status = "other"
+ if "Unreachable" in monitoring_channel['backend_status']:
+ cur_status = "Unreachable"
+ elif "defer" in monitoring_channel['backend_status']:
+ cur_status = "Deferred"
+ print 'Waiting for Monitoring_channel to be ready, current_status:%s, elapsed-time=%s' % (cur_status, time.time()-monitoringchannel_wait_start_time)
+ monitoring_channel = None
+ else:
+ print 'Monitoring_channel is ready, elapsed-time:%s' % (time.time()-monitoringchannel_wait_start_time)
+ else:
+ print 'Monitoring_channel is ready'
+ except Exception, e:
+ print 'Exception....Waiting for monitoring_channel object to be created, elapsed-time=%s' % (time.time()-monitoringchannel_wait_start_time)
+
+ if (not ceilometer_service) or (not monitoring_channel):
+ #print "Sleeping for 60 seconds...."
+ cur_attempts += 1
+ if cur_attempts > 15:
+ print "Maximum number of retrys reached....Exiting"
+ return None
+ time.sleep(60)
+ print "Attempt %s" % cur_attempts
+ else:
+ print "Both ceilometer_service and monitoring_channel are ready"
+ break
+
+ #Wait until URL is completely UP
+ while True:
+ try:
+ url = "http://localhost:8888/api/tenant/monitoring/monitoringchannel/"
+ monitoring_channel = requests.get(url, auth=admin_auth).json()[0]
+ if not monitoring_channel['ceilometer_url']:
+ print 'Waiting for monitoring channel URL to be available, elapsed-time=%s' % (e.reason,time.time()-start_time)
+ time.sleep(5)
+ pass
+ else:
+ response = urllib2.urlopen(monitoring_channel['ceilometer_url'],timeout=5)
+ break
+ except urllib2.HTTPError, e:
+ print 'HTTP error %s ...Means monitoring channel URL is reachable, elapsed-time=%s' % (e.reason,time.time()-start_time)
+ return monitoring_channel
+ except urllib2.URLError, e:
+ print 'URL error...Waiting for monitoring channle URL %s is reachable, elapsed-time=%s' % (monitoring_channel['ceilometer_url'],time.time()-start_time)
+ time.sleep(5)
+ pass
+
+#Test to verify the onboarding of monitoring service and monitoring channel
+#Test to verify there is no telemetry data available in the monitoring service initially
+def test_1():
+ global monitoring_channel
+ monitoring_channel = acquire_xos_monitoring_channel()
+ assert monitoring_channel != None
+ try:
+ url = monitoring_channel['ceilometer_url']+"v2/meters"
+ response = urllib2.urlopen(url)
+ data = json.load(response)
+ assert len(data) == 0, "Meters list is non empty for the first time"
+ print 'CURL on ceilometer URL succeeded %s' % data
+ except Exception, e:
+ print 'CURL on ceilometer URL failed %s' % e
+
+#Test to verify telemetry data from openstack and onos services is available in the monitoring service
+def test_2():
+ global monitoring_channel
+ if not monitoring_channel:
+ monitoring_channel = acquire_xos_monitoring_channel()
+ assert monitoring_channel != None
+ cur_attempts = 1
+ while True:
+ try:
+ url = monitoring_channel['ceilometer_url']+"v2/meters"
+ response = urllib2.urlopen(url,timeout=20)
+ data = json.load(response)
+ if (len(data) == 0):
+ assert (cur_attempts < 5), "Meters list can not be empty after infra monitoring is enabled....Max retries reached"
+ print 'Waiting for monitoring channle URL %s to return metrics' % (url)
+ time.sleep(10)
+ cur_attempts += 1
+ continue
+ assert any(d['name'] == 'disk.write.requests' for d in data), "Metrics does not contains disk related statistics"
+ assert any(d['name'] == 'cpu' for d in data), "Metrics does not contains cpu related statistics"
+ assert any(d['name'] == 'memory' for d in data), "Metrics does not contains memory related statistics"
+ print 'CURL on ceilometer URL succeeded Number of meters: %s' % (str(len(data)))
+ break
+ except Exception, e:
+ print 'CURL on ceilometer URL failed...%s' % e
+ break
+
+#Test to verify telemetry data from vSG services is available in the monitoring service
+def test_3():
+ global monitoring_channel
+ if not monitoring_channel:
+ monitoring_channel = acquire_xos_monitoring_channel()
+ assert monitoring_channel != None
+ cur_attempts = 1
+ while True:
+ try:
+ url = monitoring_channel['ceilometer_url']+"v2/meters"
+ response = urllib2.urlopen(url,timeout=20)
+ data = json.load(response)
+ if (len(data) == 0):
+ assert (cur_attempts < 5), "Meters list can not be empty after infra monitoring is enabled....Max retries reached"
+ print 'Waiting for monitoring channle URL %s to return metrics' % (url)
+ time.sleep(10)
+ cur_attempts += 1
+ continue
+ assert any(d['name'] == 'vsg.dns.cache.size' for d in data), "Metrics does not contains vsg.dns.cache.size related statistics"
+ assert any(d['name'] == 'vsg.dns.replaced_unexpired_entries' for d in data), "Metrics does not contains vsg.dns.replaced_unexpired_entries related statistics"
+ assert any(d['name'] == 'vsg.dns.queries_answered_locally' for d in data), "Metrics does not contains vsg.dns.queries_answered_locally related statistics"
+ print 'CURL on ceilometer URL succeeded Number of meters: %s' % (str(len(data)))
+ break
+ except Exception, e:
+ print 'CURL on ceilometer URL failed...%s' % e
+ break
+
+def usage():
+ print 'monitoringservice_test.py --test=<num>'
+
+def main(argv):
+ try:
+ opts, args = getopt.getopt(argv,"ht:",["help","test="])
+ except getopt.GetoptError:
+ usage()
+ sys.exit(2)
+
+ for opt, arg in opts:
+ if opt in ("-h", "--help"):
+ usage()
+ sys.exit()
+ elif opt in ("-t", "--test"):
+ fq = "test_"+str(arg)
+ globals()[fq]()
+
+if __name__ == "__main__":
+ main(sys.argv[1:])
diff --git a/xos/test/roles/common/tasks/main.yml b/xos/test/roles/common/tasks/main.yml
new file mode 100644
index 0000000..32c9613
--- /dev/null
+++ b/xos/test/roles/common/tasks/main.yml
@@ -0,0 +1,17 @@
+---
+- name: Validate Monitoring service is ready
+ uri:
+ url: "{{item}}"
+ user: "{{ auth.user }}"
+ password: "{{ auth.pass }}"
+ status_code: 200
+ body_format: json
+ force_basic_auth: yes
+ HEADER_Content-Type: "application/json"
+ register: monitoringservice
+ until: "monitoringservice.json and 'OK' in monitoringservice.json[0].backend_status"
+ retries: 5
+ delay: 60
+ with_items:
+ - http://localhost:8888/api/service/monitoring/
+ - http://localhost:8888/api/tenant/monitoring/monitoringchannel/
diff --git a/xos/test/roles/test-inframonitoring/tasks/main.yml b/xos/test/roles/test-inframonitoring/tasks/main.yml
new file mode 100644
index 0000000..5703e48
--- /dev/null
+++ b/xos/test/roles/test-inframonitoring/tasks/main.yml
@@ -0,0 +1,28 @@
+---
+- name: Validate Monitoring URL is completely UP and valid metrics are returned
+ uri:
+ url: "{{monitoringservice.results[1].json[0].ceilometer_url}}v2/meters"
+ status_code: 200
+ return_content: yes
+ body_format: json
+ HEADER_Content-Type: "application/json"
+ register: metrics
+ until: "(metrics.status == 200) and (metrics.content|from_json|length>0)"
+ retries: 5
+ delay: 10
+
+- name: assert disk related infra metrics are present
+ assert:
+ that: "'disk.write.requests' in metrics.content"
+ msg: "Metrics does not contains disk related statistics"
+
+- name: assert memory related infra metrics are present
+ assert:
+ that: "'memory' in metrics.content"
+ msg: "Metrics does not contains memory related statistics"
+
+- name: assert cpu related infra metrics are present
+ assert:
+ that: "'cpu' in metrics.content"
+ msg: "Metrics does not contains cpu related statistics"
+
diff --git a/xos/test/roles/test-initial/tasks/main.yml b/xos/test/roles/test-initial/tasks/main.yml
new file mode 100644
index 0000000..dea7524
--- /dev/null
+++ b/xos/test/roles/test-initial/tasks/main.yml
@@ -0,0 +1,17 @@
+---
+- name: Validate Monitoring URL is completely UP
+ uri:
+ url: "{{monitoringservice.results[1].json[0].ceilometer_url}}v2/meters"
+ status_code: 200
+ return_content: yes
+ body_format: json
+ HEADER_Content-Type: "application/json"
+ register: metrics
+ until: metrics.status == 200
+ retries: 5
+ delay: 10
+
+- name: assert no metrics are present
+ assert:
+ that: metrics.content|from_json|length==0
+ msg: "Meters list is non empty for the first time"
diff --git a/xos/test/roles/test-vsgmonitoring/tasks/main.yml b/xos/test/roles/test-vsgmonitoring/tasks/main.yml
new file mode 100644
index 0000000..11d8be6
--- /dev/null
+++ b/xos/test/roles/test-vsgmonitoring/tasks/main.yml
@@ -0,0 +1,28 @@
+---
+- name: Validate Monitoring URL is completely UP and metrics contain vSG related metrics
+ uri:
+ url: "{{monitoringservice.results[1].json[0].ceilometer_url}}v2/meters"
+ status_code: 200
+ return_content: yes
+ body_format: json
+ HEADER_Content-Type: "application/json"
+ register: metrics
+ until: "(metrics.status == 200) and (metrics.content|from_json|length>0) and ('vsg' in metrics.content)"
+ retries: 5
+ delay: 10
+
+- name: assert vSG metrics vsg.dns.cache.size are present
+ assert:
+ that: "'vsg.dns.cache.size' in metrics.content"
+ msg: "Metrics does not contains vsg.dns.cache.size related statistics"
+
+- name: assert vSG metrics vsg.dns.replaced_unexpired_entries are present
+ assert:
+ that: "'vsg.dns.replaced_unexpired_entries' in metrics.content"
+ msg: "Metrics does not contains vsg.dns.replaced_unexpired_entries related statistics"
+
+- name: assert vSG metrics vsg.dns.queries_answered_locally are present
+ assert:
+ that: "'vsg.dns.queries_answered_locally' in metrics.content"
+ msg: "Metrics does not contains vsg.dns.queries_answered_locally related statistics"
+