Test: Adding a new module of stats monitoring
using collectd tool which can be applied for
any test cases to check behaviour.
Description:
collectd is a daemon which collects system and application
stats.
Following are the changes:
1. Added changes in test container to initiate stat collection
using collectd.
2. Provided an option in cli to enable collectd for stats.
3. Also provided an option for seperate collectd container as a external
monitoring service for stats collection.
4. Multiple test cases added w.r.t collectd itself also.
Change-Id: I5017e521b18f1e20ca8cd61b07be796d8e187a3d
diff --git a/src/test/monitoring/__init__.py b/src/test/monitoring/__init__.py
new file mode 100644
index 0000000..58368ab
--- /dev/null
+++ b/src/test/monitoring/__init__.py
@@ -0,0 +1,26 @@
+#
+# Copyright 2016-present Ciena Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os,sys
+##add the python path to lookup the utils
+working_dir = os.path.dirname(os.path.realpath(sys.argv[-1]))
+utils_dir = os.path.join(working_dir, '../utils')
+fsm_dir = os.path.join(working_dir, '../fsm')
+subscriber_dir = os.path.join(working_dir, '../subscriber')
+__path__.append(utils_dir)
+__path__.append(fsm_dir)
+__path__.append(subscriber_dir)
+cli_dir = os.path.join(working_dir, '../cli')
+__path__.append(cli_dir)
diff --git a/src/test/monitoring/monitoringTest.py b/src/test/monitoring/monitoringTest.py
new file mode 100644
index 0000000..f9889de
--- /dev/null
+++ b/src/test/monitoring/monitoringTest.py
@@ -0,0 +1,329 @@
+#
+# Copyright 2016-present Ciena Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import unittest
+import time
+import os, subprocess
+from nose.tools import *
+from nose.twistedtools import reactor, deferred
+from twisted.internet import defer
+from OnosCtrl import OnosCtrl
+from scapy.all import *
+from CordContainer import *
+from docker import Client
+import json
+import requests
+log.setLevel('INFO')
+
+class monitoring_exchange(unittest.TestCase):
+
+ controllers = os.getenv('ONOS_CONTROLLER_IP', '').split(',')
+ onosLogLevel = 'INFO'
+ test_host_base = 'cord-tester1'#Hardcoded temporarily
+ collectd_app = 'org.onosproject.cpman'
+ testHostName = os.getenv('TEST_HOST', test_host_base)
+ testLogLevel = os.getenv('LOG_LEVEL', onosLogLevel)
+ stat_optionList = os.getenv('USER_OPTIONS', '').split(',')
+ serverOptionsList = os.getenv('EXTERNAL_SERVER_OPTIONS', None)
+ CBENCH_TIMEOUT = 60
+
+ @classmethod
+ def setUpClass(cls):
+ onos_ctrl = OnosCtrl('org.onosproject.cpman')
+ status, _ = onos_ctrl.activate()
+
+ @classmethod
+ def tearDownClass(cls):
+ onos_ctrl = OnosCtrl('org.onosproject.cpman')
+ status, _ = onos_ctrl.deactivate()
+
+ @classmethod
+ def stat_option(cls, stats = None, serverDetails = None):
+ # each stats option we can do some specific functions
+ if stats is None:
+ stats = cls.stat_optionList
+ if serverDetails is None:
+ serverDetails = cls.serverOptionsList
+ stats_choice = 'COLLECTD'
+ test_name = cls.testHostName
+ test_image = 'cord-test/nose'
+ if stats_choice in stats:
+ onos_ctrl = OnosCtrl('org.onosproject.cpman')
+ status, _ = onos_ctrl.activate()
+ if serverDetails is '':
+ pass
+ elif serverDetails in 'NEW':
+ test_image = 'cord-test/exserver'
+ test_name ='cord-collectd'
+ else:
+ pass
+ ## TO-DO for already up and running server, install collectd agent etc...
+ cls.start_collectd_agent_in_server(name = test_name, image = test_image)
+ return
+
+
+ @classmethod
+ def collectd_agent_metrics(cls,controller=None, auth =None, url = None):
+ '''This function is getting a rules from ONOS with json formate'''
+ if url:
+ resp = requests.get(url, auth = auth)
+ log.info('CollectD agent has provided metrics via ONOS controller, \nurl = %s \nand stats = %s \nResponse = %s ' %(url,resp.json(),resp.ok))
+ assert_equal(resp.ok, True)
+ return resp
+
+
+ @classmethod
+ def start_collectd_agent_in_server(cls, name = None, image = None):
+ container_cmd_exec = Container(name = name, image = image)
+ tty = False
+ dckr = Client()
+ cmd = 'sudo /etc/init.d/collectd start'
+ i = container_cmd_exec.execute(cmd = cmd, tty= tty, stream = True, shell = False)
+ return i
+
+ @deferred(CBENCH_TIMEOUT)
+ def test_stats_with_collectd_installation(self):
+ df = defer.Deferred()
+ def collectd_sample(df):
+ cmd = 'sudo /etc/init.d/collectd start'
+ output = subprocess.check_output(cmd,shell= True)
+ if 'Starting statistics collectio' in output:
+ log.info('Collectd is installed properly')
+ pass
+ else:
+ log.info('Collectd is not installed properly')
+ assert_equal(False, True)
+ df.callback(0)
+ reactor.callLater(0, collectd_sample, df)
+ return df
+
+ @deferred(CBENCH_TIMEOUT)
+ def test_stats_with_collectd_plugin_and_onos_installation(self):
+ df = defer.Deferred()
+ def collectd_sample(df):
+ cmd = 'ls'
+ output = subprocess.check_output(cmd,shell= True)
+ if 'write_onos' in output:
+ log.info('Collectd is installed properly and plugin happend to ONOS')
+ pass
+ else:
+ log.info('Collectd is not installed properly and no plugin happend to ONOS')
+ assert_equal(False, True)
+ df.callback(0)
+ reactor.callLater(0, collectd_sample, df)
+ return df
+
+ @deferred(CBENCH_TIMEOUT)
+ def test_stats_with_collectd_get_cpu_stats(self):
+ df = defer.Deferred()
+ def collectd_sample(df):
+ self.stat_option()
+ for controller in self.controllers:
+ if not controller:
+ continue
+ url_cpu_stats = 'http://%s:8181/onos/cpman/controlmetrics/cpu_metrics'%(controller)
+ auth = ('karaf', 'karaf')
+ self.collectd_agent_metrics(controller, auth, url = url_cpu_stats)
+ log.info('Successfully CPU metrics are retained by the stats')
+ df.callback(0)
+ reactor.callLater(0, collectd_sample, df)
+ return df
+
+ @deferred(CBENCH_TIMEOUT)
+ def test_stats_with_collectd_get_mem_stats(self):
+ df = defer.Deferred()
+ def collectd_sample(df):
+ self.stat_option()
+ for controller in self.controllers:
+ if not controller:
+ continue
+ url_mem_stats = 'http://%s:8181/onos/cpman/controlmetrics/memory_metrics'%(controller)
+ auth = ('karaf', 'karaf')
+ self.collectd_agent_metrics(controller, auth, url = url_mem_stats)
+ log.info('Successfully memory metrics are retained by the stats')
+ df.callback(0)
+ reactor.callLater(0, collectd_sample, df)
+ return df
+
+ @deferred(CBENCH_TIMEOUT)
+ def test_stats_with_collectd_get_control_metrics_messages(self):
+ df = defer.Deferred()
+ def collectd_sample(df):
+ self.stat_option()
+ for controller in self.controllers:
+ if not controller:
+ continue
+ url_messages_stats = 'http://%s:8181/onos/cpman/controlmetrics/messages'%(controller)
+ auth = ('karaf', 'karaf')
+ self.collectd_agent_metrics(controller, auth, url = url_messages_stats)
+ log.info('Successfully messages are retained by the stats')
+ df.callback(0)
+ reactor.callLater(0, collectd_sample, df)
+ return df
+
+ @deferred(CBENCH_TIMEOUT)
+ def test_stats_with_collectd_get_network_metrics_stats(self):
+ df = defer.Deferred()
+ def collectd_sample(df):
+ self.stat_option()
+ for controller in self.controllers:
+ if not controller:
+ continue
+ url_network_stats = 'http://%s:8181/onos/cpman/controlmetrics/network_metrics'%(controller)
+ auth = ('karaf', 'karaf')
+ self.collectd_agent_metrics(controller, auth, url = url_network_stats)
+ log.info('Successfully network metrics are retained by the stats')
+ df.callback(0)
+ reactor.callLater(0, collectd_sample, df)
+ return df
+
+ @deferred(CBENCH_TIMEOUT)
+ def test_stats_with_collectd_get_network_metrics_stats(self):
+ df = defer.Deferred()
+ def collectd_sample(df):
+ self.stat_option()
+ for controller in self.controllers:
+ if not controller:
+ continue
+ url_network_stats = 'http://%s:8181/onos/cpman/controlmetrics/disk_metrics'%(controller)
+ auth = ('karaf', 'karaf')
+ self.collectd_agent_metrics(controller, auth, url = url_network_stats)
+ log.info('Successfully disk metrics are retained by the stats')
+ df.callback(0)
+ reactor.callLater(0, collectd_sample, df)
+ return df
+
+ @deferred(CBENCH_TIMEOUT)
+ def test_stats_with_collectd_for_installing_new_container(self):
+ df = defer.Deferred()
+ def collectd_sample(df):
+ if 'NEW' in self.serverOptionsList:
+ test_image = 'cord-test/exserver'
+ test_name ='cord-collectd'
+ ## stopping collectd agent on test container if any
+ cmd = 'sudo /etc/init.d/collectd stop'
+ output = os.system(cmd)
+ ## starting collectd agent on new container
+ cmd = 'sudo /etc/init.d/collectd start'
+ output = self.start_collectd_agent_in_server(name = test_name, image = test_image)
+ if output == 0:
+ log.info('Collectd is installed properly on new container')
+ pass
+ else:
+ log.info('Collectd is not installed properly on new container')
+ assert_equal(False, True)
+ df.callback(0)
+ reactor.callLater(0, collectd_sample, df)
+ return df
+
+ @deferred(CBENCH_TIMEOUT)
+ def test_stats_with_collectd_for_cpu_metrics_on_new_container(self):
+ df = defer.Deferred()
+ def collectd_sample(df):
+ if 'NEW' in self.serverOptionsList:
+ ## stopping collectd agent on test container if any
+ cmd = 'sudo /etc/init.d/collectd stop'
+ output = os.system(cmd)
+ self.stat_option()
+ for controller in self.controllers:
+ if not controller:
+ continue
+ url_cpu_stats = 'http://%s:8181/onos/cpman/controlmetrics/cpu_metrics'%(controller)
+ auth = ('karaf', 'karaf')
+ self.collectd_agent_metrics(controller, auth, url = url_cpu_stats)
+ log.info('Successfully CPU metrics are retained by the stats')
+ df.callback(0)
+ reactor.callLater(0, collectd_sample, df)
+ return df
+
+ @deferred(CBENCH_TIMEOUT)
+ def test_stats_with_collectd_memory_metrics_on_new_container(self):
+ df = defer.Deferred()
+ def collectd_sample(df):
+ if 'NEW' in self.serverOptionsList:
+ ## stopping collectd agent on test container if any
+ cmd = 'sudo /etc/init.d/collectd stop'
+ output = os.system(cmd)
+ self.stat_option()
+ for controller in self.controllers:
+ if not controller:
+ continue
+ url_mem_stats = 'http://%s:8181/onos/cpman/controlmetrics/memory_metrics'%(controller)
+ auth = ('karaf', 'karaf')
+ self.collectd_agent_metrics(controller, auth, url = url_mem_stats)
+ log.info('Successfully memory metrics are retained by the stats')
+ df.callback(0)
+ reactor.callLater(0, collectd_sample, df)
+ return df
+
+ @deferred(CBENCH_TIMEOUT)
+ def test_stats_with_collectd_get_messages_on_new_container(self):
+ df = defer.Deferred()
+ def collectd_sample(df):
+ if 'NEW' in self.serverOptionsList:
+ ## stopping collectd agent on test container if any
+ cmd = 'sudo /etc/init.d/collectd stop'
+ output = os.system(cmd)
+ self.stat_option()
+ for controller in self.controllers:
+ if not controller:
+ continue
+ url_messages_stats = 'http://%s:8181/onos/cpman/controlmetrics/messages'%(controller)
+ auth = ('karaf', 'karaf')
+ self.collectd_agent_metrics(controller, auth, url = url_messages_stats)
+ log.info('Successfully messages metrics are retained by the stats')
+ df.callback(0)
+ reactor.callLater(0, collectd_sample, df)
+ return df
+
+ @deferred(CBENCH_TIMEOUT)
+ def test_stats_with_collectd_network_metrics_on_new_container(self):
+ df = defer.Deferred()
+ def collectd_sample(df):
+ if 'NEW' in self.serverOptionsList:
+ ## stopping collectd agent on test container if any
+ cmd = 'sudo /etc/init.d/collectd stop'
+ output = os.system(cmd)
+ self.stat_option()
+ for controller in self.controllers:
+ if not controller:
+ continue
+ url_network_stats = 'http://%s:8181/onos/cpman/controlmetrics/network_metrics'%(controller)
+ auth = ('karaf', 'karaf')
+ self.collectd_agent_metrics(controller, auth, url = url_network_stats)
+ log.info('Successfully network metrics are retained by the stats')
+ df.callback(0)
+ reactor.callLater(0, collectd_sample, df)
+ return df
+
+ @deferred(CBENCH_TIMEOUT)
+ def test_stats_with_collectd_disk_metrics_on_new_container(self):
+ df = defer.Deferred()
+ def collectd_sample(df):
+ if 'NEW' in self.serverOptionsList:
+ ## stopping collectd agent on test container if any
+ cmd = 'sudo /etc/init.d/collectd stop'
+ output = os.system(cmd)
+ self.stat_option()
+ for controller in self.controllers:
+ if not controller:
+ continue
+ url_disk_stats = 'http://%s:8181/onos/cpman/controlmetrics/disk_metrics'%(controller)
+ auth = ('karaf', 'karaf')
+ self.collectd_agent_metrics(controller, auth, url = url_disk_stats)
+ log.info('Successfully network metrics are retained by the stats')
+ df.callback(0)
+ reactor.callLater(0, collectd_sample, df)
+ return df
diff --git a/src/test/utils/CordLogger.py b/src/test/utils/CordLogger.py
index bb9e571..cc1845b 100644
--- a/src/test/utils/CordLogger.py
+++ b/src/test/utils/CordLogger.py
@@ -1,6 +1,11 @@
from OnosLog import OnosLog
from scapy.all import log
from onosclidriver import OnosCliDriver
+from OnosCtrl import OnosCtrl, get_mac
+from docker import Client
+from CordContainer import *
+import json
+import requests
import unittest
import os
import time
@@ -22,7 +27,7 @@
onosLogLevel = 'INFO'
curLogLevel = onosLogLevel
testLogLevel = os.getenv('LOG_LEVEL', onosLogLevel)
- setup_dir = os.path.join( os.path.dirname(os.path.realpath(__file__)), '../setup')
+ setup_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../setup')
archive_dir = os.path.join(setup_dir, 'test_logs')
onos_data_dir = os.path.join(setup_dir, 'cord-onos-data')
@@ -155,3 +160,66 @@
except:
pass
cls.cliSessionExit()
+
+ @classmethod
+ def stat_option(cls, stat = None, serverDetails = None):
+ # each stat option we can do some specific functions
+ if stat is None:
+ stat = cls.statOptionsList
+ if serverDetails is None:
+ serverDetails = cls.serverOptionsList
+ stat_choice = 'COLLECTD'
+ test_name = cls.testHostName
+ test_image = 'cord-test/nose'
+ if stat_choice in stat:
+ onos_ctrl = OnosCtrl('org.onosproject.cpman')
+ status, _ = onos_ctrl.activate()
+ if serverDetails is '':
+ ## default Test Container is used to install CollectD
+ pass
+ elif serverDetails in 'NEW':
+ test_image = 'cord-test/exserver'
+ test_name ='cord-collectd'
+ else:
+ pass
+ # cls.connect_server(serverDetails)
+ ## TO-DO for already up and running server, install collectd agent etc...
+ cls.start_collectd_agent_in_server(name = test_name, image = test_image)
+ for controller in cls.controllers:
+ if not controller:
+ continue
+ url_mem_stats = 'http://%s:8181/onos/cpman/controlmetrics/memory_metrics'%(controller)
+ url_cpu_stats = 'http://%s:8181/onos/cpman/controlmetrics/cpu_metrics'%(controller)
+ auth = ('karaf', 'karaf')
+ cls.collectd_agent_metrics(controller, auth, url = url_cpu_stats)
+ cls.collectd_agent_metrics(controller, auth, url = url_mem_stats)
+ return
+
+
+ @classmethod
+ def collectd_agent_metrics(cls,controller=None, auth =None, url = None):
+ '''This function is getting rules from ONOS with json format'''
+ if url:
+ resp = requests.get(url, auth = auth)
+ log.info('Collectd agent has provided metrics via ONOS controller, url = %s \nand status = %s' %(url,resp.json()))
+ return resp
+
+
+ @classmethod
+ def start_collectd_agent_in_server(cls, name = None, image = None):
+ container_cmd_exec = Container(name = name, image = image)
+ tty = False
+ dckr = Client()
+ cmd = 'sudo /etc/init.d/collectd start'
+ i = container_cmd_exec.execute(cmd = cmd, tty= tty, stream = True)
+ return
+
+ @classmethod
+ def disable_onos_apps(cls, stat = None, app = None):
+ stat_choice = 'COLLECTD'
+ if stat is None:
+ stat = cls.statOptionsList
+ if stat_choice in stat:
+ onos_ctrl = OnosCtrl('org.onosproject.cpman')
+ status, _ = onos_ctrl.deactivate()
+