blob: 4f76d436c27c59863156507d95376716498156a7 [file] [log] [blame]
Chetan Gaonkercb122cc2016-05-10 10:58:34 -07001#!/usr/bin/env python
A.R Karthick95d044e2016-06-10 18:44:36 -07002#
Chetan Gaonkercfcce782016-05-10 10:10:42 -07003# Copyright 2016-present Ciena Corporation
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
A.R Karthick95d044e2016-06-10 18:44:36 -07008#
Chetan Gaonkercfcce782016-05-10 10:10:42 -07009# http://www.apache.org/licenses/LICENSE-2.0
A.R Karthick95d044e2016-06-10 18:44:36 -070010#
Chetan Gaonkercfcce782016-05-10 10:10:42 -070011# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
Chetan Gaonker93e302d2016-04-05 10:51:07 -070017from argparse import ArgumentParser
A R Karthick14118c62016-07-27 14:54:04 -070018import os,sys,time,socket,errno
ChetanGaonkereadad482016-08-26 01:21:47 -070019import shutil, platform, re
Chetan Gaonker4d842ad2016-04-26 10:04:24 -070020utils_dir = os.path.join( os.path.dirname(os.path.realpath(__file__)), '../utils')
Chetan Gaonker7142a342016-04-07 14:53:12 -070021sys.path.append(utils_dir)
ChetanGaonker68a047f2016-10-12 10:31:48 -070022sys.path.insert(1, '/usr/local/lib/python2.7/dist-packages')
Chetan Gaonker93e302d2016-04-05 10:51:07 -070023from OnosCtrl import OnosCtrl
Chetan Gaonker4ca5cca2016-04-11 13:59:35 -070024from OltConfig import OltConfig
A R Karthick1f03e912016-05-18 11:39:22 -070025from threadPool import ThreadPool
Chetan Gaonker3533faa2016-04-25 17:50:14 -070026from CordContainer import *
A R Karthicke99ab5c2016-09-30 13:59:57 -070027from CordTestServer import cord_test_server_start,cord_test_server_stop,cord_test_server_shutdown,CORD_TEST_HOST,CORD_TEST_PORT
A R Karthick07608ef2016-08-23 16:51:19 -070028from TestManifest import TestManifest
ChetanGaonkereadad482016-08-26 01:21:47 -070029from docker import Client
30from docker.utils import kwargs_from_env
A R Karthickea8bfce2016-10-13 16:32:07 -070031from Xos import XosServiceProfile
A R Karthick07608ef2016-08-23 16:51:19 -070032try:
33 from Fabric import FabricMAAS
34except:
35 FabricMAAS = None
Chetan Gaonker93e302d2016-04-05 10:51:07 -070036
Chetan Gaonker93e302d2016-04-05 10:51:07 -070037class CordTester(Container):
Chetan Gaonker93e302d2016-04-05 10:51:07 -070038 sandbox = '/root/test'
Chetan Gaonker7142a342016-04-07 14:53:12 -070039 sandbox_setup = '/root/test/src/test/setup'
Chetan Gaonker4d842ad2016-04-26 10:04:24 -070040 tester_base = os.path.dirname(os.path.realpath(__file__))
41 tester_paths = os.path.realpath(__file__).split(os.path.sep)
A R Karthickb7e80902016-05-17 09:38:31 -070042 tester_path_index = tester_paths.index('src') - 1
Chetan Gaonker7142a342016-04-07 14:53:12 -070043 sandbox_host = os.path.sep.join(tester_paths[:tester_path_index+1])
Chetan Gaonker93e302d2016-04-05 10:51:07 -070044
45 host_guest_map = ( (sandbox_host, sandbox),
Chetan Gaonker85b7bd52016-04-20 10:29:12 -070046 ('/lib/modules', '/lib/modules'),
47 ('/var/run/docker.sock', '/var/run/docker.sock')
Chetan Gaonker93e302d2016-04-05 10:51:07 -070048 )
49 basename = 'cord-tester'
A R Karthick36cfcef2016-08-18 15:20:07 -070050 switch_on_olt = False
Chetan Gaonker503032a2016-05-12 12:06:29 -070051 IMAGE = 'cord-test/nose'
ChetanGaonker2c0e9bb2016-09-21 13:38:37 -070052 ALL_TESTS = ('tls', 'dhcp', 'dhcprelay','igmp', 'subscriber',
53 'cordSubscriber', 'vrouter', 'flows', 'proxyarp', 'acl', 'xos', 'fabric',
ChetanGaonker3441faf2016-10-27 12:15:21 -070054 'cbench', 'cluster', 'netCondition')
Chetan Gaonker93e302d2016-04-05 10:51:07 -070055
A R Karthicka013a272016-08-16 16:40:19 -070056 def __init__(self, tests, instance = 0, num_instances = 1, ctlr_ip = None,
A R Karthick07608ef2016-08-23 16:51:19 -070057 name = '', image = IMAGE, prefix = '', tag = 'candidate',
Chetan Gaonker85b7bd52016-04-20 10:29:12 -070058 env = None, rm = False, update = False):
A R Karthick1f03e912016-05-18 11:39:22 -070059 self.tests = tests
Chetan Gaonker4ca5cca2016-04-11 13:59:35 -070060 self.ctlr_ip = ctlr_ip
Chetan Gaonker93e302d2016-04-05 10:51:07 -070061 self.rm = rm
A R Karthicka013a272016-08-16 16:40:19 -070062 self.name = name or self.get_name()
A R Karthick07608ef2016-08-23 16:51:19 -070063 super(CordTester, self).__init__(self.name, image = image, prefix = prefix, tag = tag)
Chetan Gaonker93e302d2016-04-05 10:51:07 -070064 host_config = self.create_host_config(host_guest_map = self.host_guest_map, privileged = True)
65 volumes = []
Chetan Gaonkerb84835f2016-04-19 15:12:10 -070066 for _, g in self.host_guest_map:
Chetan Gaonker93e302d2016-04-05 10:51:07 -070067 volumes.append(g)
Chetan Gaonker85b7bd52016-04-20 10:29:12 -070068 if update is True or not self.img_exists():
A R Karthick07608ef2016-08-23 16:51:19 -070069 self.build_image(self.image_name)
A R Karthicka013a272016-08-16 16:40:19 -070070 self.create = True
71 #check if are trying to run tests on existing container
72 if not name or not self.exists():
73 ##Remove test container if any
74 self.remove_container(self.name, force=True)
75 else:
76 self.create = False
A R Karthick078e63a2016-07-28 13:59:31 -070077 self.olt = False
Chetan Gaonker4ca5cca2016-04-11 13:59:35 -070078 if env is not None and env.has_key('OLT_CONFIG'):
79 self.olt = True
A R Karthick078e63a2016-07-28 13:59:31 -070080 olt_conf_file = os.path.join(self.tester_base, 'olt_config.json')
81 olt_config = OltConfig(olt_conf_file)
82 self.port_map, _ = olt_config.olt_port_map()
83 #Try using the host interface in olt conf to setup the switch
84 if self.port_map.has_key('host'):
85 self.switch = self.port_map['host']
Chetan Gaonker4ca5cca2016-04-11 13:59:35 -070086 else:
A R Karthick078e63a2016-07-28 13:59:31 -070087 self.switch = 'ovsbr0'
A R Karthick1f03e912016-05-18 11:39:22 -070088 if env is not None:
A R Karthick078e63a2016-07-28 13:59:31 -070089 env['TEST_SWITCH'] = self.switch
A R Karthick1f03e912016-05-18 11:39:22 -070090 env['TEST_HOST'] = self.name
91 env['TEST_INSTANCE'] = instance
92 env['TEST_INSTANCES'] = num_instances
A R Karthicka013a272016-08-16 16:40:19 -070093 if self.create:
94 print('Starting test container %s, image %s, tag %s' %(self.name, self.image, self.tag))
95 self.start(rm = False, volumes = volumes, environment = env,
96 host_config = host_config, tty = True)
Chetan Gaonker4ca5cca2016-04-11 13:59:35 -070097
A.R Karthicke4631062016-11-03 14:28:19 -070098 def execute_switch(self, cmd, shell = False):
Chetan Gaonker4ca5cca2016-04-11 13:59:35 -070099 if self.olt:
100 return os.system(cmd)
101 return self.execute(cmd, shell = shell)
102
A R Karthick078e63a2016-07-28 13:59:31 -0700103 def start_switch(self, boot_delay = 2):
Chetan Gaonker4ca5cca2016-04-11 13:59:35 -0700104 """Start OVS"""
105 ##Determine if OVS has to be started locally or not
106 s_file,s_sandbox = ('of-bridge-local.sh',self.tester_base) if self.olt else ('of-bridge.sh',self.sandbox_setup)
A R Karthick078e63a2016-07-28 13:59:31 -0700107 ovs_cmd = os.path.join(s_sandbox, s_file) + ' {0}'.format(self.switch)
Chetan Gaonker4ca5cca2016-04-11 13:59:35 -0700108 if self.olt:
A R Karthick36cfcef2016-08-18 15:20:07 -0700109 if CordTester.switch_on_olt is True:
110 return
111 CordTester.switch_on_olt = True
Chetan Gaonker4ca5cca2016-04-11 13:59:35 -0700112 ovs_cmd += ' {0}'.format(self.ctlr_ip)
A R Karthick2b93d6a2016-09-06 15:19:09 -0700113 print('Starting OVS on the host with controller: %s' %(self.ctlr_ip))
Chetan Gaonker4ca5cca2016-04-11 13:59:35 -0700114 else:
115 print('Starting OVS on test container %s' %self.name)
116 self.execute_switch(ovs_cmd)
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700117 status = 1
118 ## Wait for the LLDP flows to be added to the switch
119 tries = 0
A R Karthick51e6fd82016-11-22 14:39:19 -0800120 while status != 0 and tries < 500:
A R Karthick078e63a2016-07-28 13:59:31 -0700121 cmd = 'sudo ovs-ofctl dump-flows {0} | grep \"type=0x8942\"'.format(self.switch)
Chetan Gaonker4ca5cca2016-04-11 13:59:35 -0700122 status = self.execute_switch(cmd, shell = True)
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700123 tries += 1
124 if tries % 10 == 0:
125 print('Waiting for test switch to be connected to ONOS controller ...')
126
127 if status != 0:
128 print('Test Switch not connected to ONOS container.'
129 'Please remove ONOS container and restart the test')
130 if self.rm:
131 self.kill()
132 sys.exit(1)
133
Chetan Gaonker4ca5cca2016-04-11 13:59:35 -0700134 if boot_delay:
135 time.sleep(boot_delay)
136
A R Karthick1f03e912016-05-18 11:39:22 -0700137 def setup_intfs(self, port_num = 0):
Chetan Gaonker4ca5cca2016-04-11 13:59:35 -0700138 tester_intf_subnet = '192.168.100'
139 res = 0
Chetan Gaonker5209fe82016-04-19 10:09:53 -0700140 host_intf = self.port_map['host']
141 start_vlan = self.port_map['start_vlan']
A R Karthick07769362016-07-28 17:36:15 -0700142 start_vlan += port_num
143 uplink = self.port_map['uplink']
144 wan = self.port_map['wan']
A R Karthick36cfcef2016-08-18 15:20:07 -0700145 port_list = self.port_map['ports'] + self.port_map['relay_ports']
146 for port in port_list:
Chetan Gaonker5209fe82016-04-19 10:09:53 -0700147 guest_if = port
A R Karthick1f03e912016-05-18 11:39:22 -0700148 local_if = '{0}_{1}'.format(guest_if, port_num+1)
149 guest_ip = '{0}.{1}/24'.format(tester_intf_subnet, port_num+1)
Chetan Gaonker5209fe82016-04-19 10:09:53 -0700150 ##Use pipeworks to configure container interfaces on host/bridge interfaces
A R Karthick07769362016-07-28 17:36:15 -0700151 pipework_cmd = 'pipework {0} -i {1} -l {2} {3} {4}'.format(host_intf, guest_if,
152 local_if, self.name, guest_ip)
153 #if the wan interface is specified for uplink, then use it instead
154 if wan and port == self.port_map[uplink]:
155 pipework_cmd = 'pipework {0} -i {1} -l {2} {3} {4}'.format(wan, guest_if,
156 local_if, self.name, guest_ip)
157 else:
158 if start_vlan != 0:
159 pipework_cmd += ' @{}'.format(start_vlan)
160 start_vlan += 1
A.R Karthick95d044e2016-06-10 18:44:36 -0700161
Chetan Gaonker4ca5cca2016-04-11 13:59:35 -0700162 res += os.system(pipework_cmd)
Chetan Gaonker5209fe82016-04-19 10:09:53 -0700163 port_num += 1
Chetan Gaonker4ca5cca2016-04-11 13:59:35 -0700164
A R Karthick1f03e912016-05-18 11:39:22 -0700165 return res, port_num
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700166
167 @classmethod
A R Karthickb50f5592016-07-26 12:19:29 -0700168 def cleanup_intfs(cls):
169 olt_conf_file = os.path.join(cls.tester_base, 'olt_config.json')
170 olt_config = OltConfig(olt_conf_file)
A R Karthickb03cecd2016-07-27 10:27:55 -0700171 port_map, _ = olt_config.olt_port_map()
A R Karthickb50f5592016-07-26 12:19:29 -0700172 port_num = 0
173 intf_host = port_map['host']
174 start_vlan = port_map['start_vlan']
A R Karthick07769362016-07-28 17:36:15 -0700175 uplink = port_map['uplink']
176 wan = port_map['wan']
A R Karthickb50f5592016-07-26 12:19:29 -0700177 intf_type = 0
178 if os.path.isdir('/sys/class/net/{}/bridge'.format(intf_host)):
179 intf_type = 1 ##linux bridge
180 else:
181 cmd = 'ovs-vsctl list-br | grep -q "^{0}$"'.format(intf_host)
182 res = os.system(cmd)
183 if res == 0: ##ovs bridge
184 intf_type = 2
185 cmds = ()
186 res = 0
A R Karthick0ff033f2016-11-16 10:47:30 -0800187 for port in port_map['ports'] + port_map['relay_ports']:
A R Karthickb50f5592016-07-26 12:19:29 -0700188 local_if = '{0}_{1}'.format(port, port_num+1)
189 if intf_type == 0:
190 if start_vlan != 0:
A R Karthick07769362016-07-28 17:36:15 -0700191 cmds = ('ip link del {}.{}'.format(intf_host, start_vlan),)
192 start_vlan += 1
A R Karthickb50f5592016-07-26 12:19:29 -0700193 else:
194 if intf_type == 1:
195 cmds = ('brctl delif {} {}'.format(intf_host, local_if),
196 'ip link del {}'.format(local_if))
197 else:
A.R Karthicke4631062016-11-03 14:28:19 -0700198 cmds = ('ovs-vsctl del-port {} {}'.format(intf_host, local_if),
199 'ip link del {}'.format(local_if))
A R Karthickb50f5592016-07-26 12:19:29 -0700200
201 for cmd in cmds:
202 res += os.system(cmd)
203
204 port_num += 1
205
206 @classmethod
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700207 def get_name(cls):
208 cnt_name = '/{0}'.format(cls.basename)
209 cnt_name_len = len(cnt_name)
210 names = list(flatten(n['Names'] for n in cls.dckr.containers(all=True)))
211 test_names = filter(lambda n: n.startswith(cnt_name), names)
212 last_cnt_number = 0
213 if test_names:
214 last_cnt_name = reduce(lambda n1, n2: n1 if int(n1[cnt_name_len:]) > \
215 int(n2[cnt_name_len:]) else n2,
216 test_names)
217 last_cnt_number = int(last_cnt_name[cnt_name_len:])
218 test_cnt_name = cls.basename + str(last_cnt_number+1)
219 return test_cnt_name
220
221 @classmethod
222 def build_image(cls, image):
223 print('Building test container docker image %s' %image)
Chetan Gaonkerb6064fa2016-05-02 16:29:57 -0700224 ovs_version = '2.5.0'
225 image_format = (ovs_version,)*4
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700226 dockerfile = '''
227FROM ubuntu:14.04
228MAINTAINER chetan@ciena.com
A R Karthickc762df42016-05-25 10:09:21 -0700229RUN apt-get update && \
230 apt-get install -y git git-core autoconf automake autotools-dev pkg-config \
231 make gcc g++ libtool libc6-dev cmake libpcap-dev libxerces-c2-dev \
232 unzip libpcre3-dev flex bison libboost-dev \
233 python python-pip python-setuptools python-scapy tcpdump doxygen doxypy wget \
234 openvswitch-common openvswitch-switch \
A R Karthick07608ef2016-08-23 16:51:19 -0700235 python-twisted python-sqlite sqlite3 python-pexpect telnet arping isc-dhcp-server \
236 python-paramiko python-maas-client
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700237RUN easy_install nose
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700238RUN mkdir -p /root/ovs
239WORKDIR /root
Chetan Gaonkerb6064fa2016-05-02 16:29:57 -0700240RUN wget http://openvswitch.org/releases/openvswitch-{}.tar.gz -O /root/ovs/openvswitch-{}.tar.gz && \
241(cd /root/ovs && tar zxpvf openvswitch-{}.tar.gz && \
242 cd openvswitch-{} && \
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700243 ./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --disable-ssl && make && make install)
244RUN service openvswitch-switch restart || /bin/true
A.R Karthickec5b72a2016-11-03 09:53:07 -0700245RUN pip install scapy==2.3.2 scapy-ssl_tls==1.2.2 monotonic configObj docker-py pyyaml nsenter pyroute2 netaddr python-daemon
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700246RUN mv /usr/sbin/tcpdump /sbin/
247RUN ln -sf /sbin/tcpdump /usr/sbin/tcpdump
A R Karthickf4999472016-07-01 16:42:13 -0700248RUN mv /usr/sbin/dhcpd /sbin/
249RUN ln -sf /sbin/dhcpd /usr/sbin/dhcpd
A R Karthickb7e80902016-05-17 09:38:31 -0700250WORKDIR /root
251RUN wget -nc http://de.archive.ubuntu.com/ubuntu/pool/main/b/bison/bison_2.5.dfsg-2.1_amd64.deb \
252 http://de.archive.ubuntu.com/ubuntu/pool/main/b/bison/libbison-dev_2.5.dfsg-2.1_amd64.deb
253RUN sudo dpkg -i bison_2.5.dfsg-2.1_amd64.deb libbison-dev_2.5.dfsg-2.1_amd64.deb
254RUN rm bison_2.5.dfsg-2.1_amd64.deb libbison-dev_2.5.dfsg-2.1_amd64.deb
255RUN wget -nc http://www.nbee.org/download/nbeesrc-jan-10-2013.zip && \
256 unzip nbeesrc-jan-10-2013.zip && \
257 cd nbeesrc-jan-10-2013/src && cmake . && make && \
258 cp ../bin/libn*.so /usr/local/lib && ldconfig && \
259 cp -R ../include/* /usr/include/
260WORKDIR /root
261RUN git clone https://github.com/CPqD/ofsoftswitch13.git && \
262 cd ofsoftswitch13 && \
A R Karthickb7e80902016-05-17 09:38:31 -0700263 ./boot.sh && \
264 ./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --disable-ssl && \
265 make && make install
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700266CMD ["/bin/bash"]
Chetan Gaonkerb6064fa2016-05-02 16:29:57 -0700267'''.format(*image_format)
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700268 super(CordTester, cls).build_image(dockerfile, image)
269 print('Done building docker image %s' %image)
270
A R Karthick1f03e912016-05-18 11:39:22 -0700271 def run_tests(self):
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700272 '''Run the list of tests'''
A R Karthick9a5edc42016-08-24 19:10:22 -0700273 res = 0
A R Karthick1f03e912016-05-18 11:39:22 -0700274 print('Running tests: %s' %self.tests)
275 for t in self.tests:
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700276 test = t.split(':')[0]
A R Karthick24f1de62016-05-12 15:16:38 -0700277 test_file = '{}Test.py'.format(test)
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700278 if t.find(':') >= 0:
A R Karthick24f1de62016-05-12 15:16:38 -0700279 test_case = '{0}:{1}'.format(test_file, t.split(':')[1])
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700280 else:
281 test_case = test_file
Chetan Gaonker7142a342016-04-07 14:53:12 -0700282 cmd = 'nosetests -v {0}/src/test/{1}/{2}'.format(self.sandbox, test, test_case)
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700283 status = self.execute(cmd, shell = True)
A R Karthick9a5edc42016-08-24 19:10:22 -0700284 if status > 255:
285 status = 1
286 res |= status
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700287 print('Test %s %s' %(test_case, 'Success' if status == 0 else 'Failure'))
288 print('Done running tests')
289 if self.rm:
290 print('Removing test container %s' %self.name)
291 self.kill(remove=True)
292
A R Karthick9a5edc42016-08-24 19:10:22 -0700293 return res
294
Chetan Gaonkerfb3cb5e2016-05-06 11:55:44 -0700295 @classmethod
296 def list_tests(cls, tests):
297 print('Listing test cases')
298 for test in tests:
A R Karthick24f1de62016-05-12 15:16:38 -0700299 test_file = '{}Test.py'.format(test)
Chetan Gaonkerfb3cb5e2016-05-06 11:55:44 -0700300 cmd = 'nosetests -v --collect-only {0}/../{1}/{2}'.format(cls.tester_base, test, test_file)
301 os.system(cmd)
302
A R Karthicka013a272016-08-16 16:40:19 -0700303
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700304##default onos/radius/test container images and names
305onos_image_default='onosproject/onos:latest'
A R Karthick07608ef2016-08-23 16:51:19 -0700306nose_image_default= '{}:candidate'.format(CordTester.IMAGE)
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700307test_type_default='dhcp'
A.R Karthick95d044e2016-06-10 18:44:36 -0700308onos_app_version = '2.0-SNAPSHOT'
Chetan Gaonker4d842ad2016-04-26 10:04:24 -0700309cord_tester_base = os.path.dirname(os.path.realpath(__file__))
Chetan Gaonker4ca5cca2016-04-11 13:59:35 -0700310onos_app_file = os.path.abspath('{0}/../apps/ciena-cordigmp-'.format(cord_tester_base) + onos_app_version + '.oar')
A R Karthick81acbff2016-06-17 14:45:16 -0700311cord_test_server_address = '{}:{}'.format(CORD_TEST_HOST, CORD_TEST_PORT)
A R Karthick07608ef2016-08-23 16:51:19 -0700312identity_file_default = '/etc/maas/ansible/id_rsa'
313
314##sets up the ssh key file for the test container
315def set_ssh_key_file(identity_file):
316 ssh_key_file = None
317 if os.access(identity_file, os.F_OK):
318 ##copy it to setup directory
319 identity_dest = os.path.join(CordTester.tester_base, 'id_rsa')
320 if os.path.abspath(identity_file) != identity_dest:
321 try:
322 shutil.copy(identity_file, identity_dest)
323 ssh_key_file = os.path.join(CordTester.sandbox_setup, 'id_rsa')
324 except: pass
325
326 return ssh_key_file
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700327
328def runTest(args):
Chetan Gaonker823cdc52016-05-09 15:51:23 -0700329 #Start the cord test tcp server
A R Karthick81acbff2016-06-17 14:45:16 -0700330 test_server_params = args.server.split(':')
331 test_host = test_server_params[0]
332 test_port = CORD_TEST_PORT
333 if len(test_server_params) > 1:
334 test_port = int(test_server_params[1])
335 try:
336 test_server = cord_test_server_start(daemonize = False, cord_test_host = test_host, cord_test_port = test_port)
337 except:
338 ##Most likely a server instance is already running (daemonized earlier)
339 test_server = None
340
A R Karthick1f03e912016-05-18 11:39:22 -0700341 test_containers = []
342 #These tests end up restarting ONOS/quagga/radius
A R Karthick4e0c0912016-08-17 16:57:42 -0700343 tests_exempt = ('vrouter', 'cordSubscriber', 'proxyarp', 'dhcprelay')
Chetan Gaonker503032a2016-05-12 12:06:29 -0700344 if args.test_type.lower() == 'all':
345 tests = CordTester.ALL_TESTS
Chetan Gaonker503032a2016-05-12 12:06:29 -0700346 args.quagga = True
347 else:
A R Karthickacae3b42016-05-12 15:27:24 -0700348 tests = args.test_type.split('-')
Chetan Gaonker503032a2016-05-12 12:06:29 -0700349
A R Karthick1f03e912016-05-18 11:39:22 -0700350 tests_parallel = [ t for t in tests if t.split(':')[0] not in tests_exempt ]
351 tests_not_parallel = [ t for t in tests if t.split(':')[0] in tests_exempt ]
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700352 onos_cnt = {'tag':'latest'}
A R Karthick07608ef2016-08-23 16:51:19 -0700353 nose_cnt = {'image': CordTester.IMAGE, 'tag': 'candidate'}
Chetan Gaonker503032a2016-05-12 12:06:29 -0700354 update_map = { 'quagga' : False, 'test' : False, 'radius' : False }
355 update_map[args.update.lower()] = True
A.R Karthick95d044e2016-06-10 18:44:36 -0700356
Chetan Gaonker503032a2016-05-12 12:06:29 -0700357 if args.update.lower() == 'all':
358 for c in update_map.keys():
359 update_map[c] = True
A.R Karthick95d044e2016-06-10 18:44:36 -0700360
A R Karthick07608ef2016-08-23 16:51:19 -0700361 onos_ip = None
Chetan Gaonker5209fe82016-04-19 10:09:53 -0700362 radius_ip = None
A R Karthick07608ef2016-08-23 16:51:19 -0700363 head_node = platform.node()
364 use_manifest = False
365 if args.manifest:
366 if os.access(args.manifest, os.F_OK):
367 ##copy it to setup directory
368 dest = os.path.join(CordTester.tester_base, 'manifest.json')
369 if os.path.abspath(args.manifest) != dest:
370 try:
371 shutil.copy(args.manifest, dest)
372 except: pass
373 test_manifest = TestManifest(dest)
374 onos_ip = test_manifest.onos_ip
375 radius_ip = test_manifest.radius_ip
376 head_node = test_manifest.head_node
377 use_manifest = True
378 else:
379 print('Unable to access test manifest: %s' %args.manifest)
Chetan Gaonkerfb3cb5e2016-05-06 11:55:44 -0700380
Chetan Gaonker5209fe82016-04-19 10:09:53 -0700381 #don't spawn onos if the user has specified external test controller with test interface config
382 if args.test_controller:
383 ips = args.test_controller.split('/')
384 onos_ip = ips[0]
385 if len(ips) > 1:
386 radius_ip = ips[1]
387 else:
388 radius_ip = None
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700389
A R Karthick07608ef2016-08-23 16:51:19 -0700390 Container.IMAGE_PREFIX = args.prefix
ChetanGaonkerdbd4e4b2016-10-28 17:40:11 -0700391 cluster_mode = True if args.onos_instances > 1 else False
A R Karthick1ef70552016-11-17 17:33:36 -0800392 existing_list = [ c['Names'][0][1:] for c in Container.dckr.containers() if c['Image'] == args.onos ]
393 setup_cluster = False if len(existing_list) == args.onos_instances else True
A R Karthick07608ef2016-08-23 16:51:19 -0700394 if onos_ip is None:
395 image_names = args.onos.rsplit(':', 1)
396 onos_cnt['image'] = image_names[0]
397 if len(image_names) > 1:
398 if image_names[1].find('/') < 0:
399 onos_cnt['tag'] = image_names[1]
400 else:
401 #tag cannot have slashes
402 onos_cnt['image'] = args.onos
403
404 Onos.IMAGE = onos_cnt['image']
405 Onos.PREFIX = args.prefix
406 Onos.TAG = onos_cnt['tag']
A R Karthick09dbc6d2016-11-22 10:37:42 -0800407 data_volume = '{}-data'.format(Onos.NAME) if args.shared_volume else None
A R Karthick07608ef2016-08-23 16:51:19 -0700408 onos = Onos(image = Onos.IMAGE,
A R Karthickec2db322016-11-17 15:06:01 -0800409 tag = Onos.TAG, boot_delay = 60, cluster = cluster_mode, data_volume = data_volume)
Chetan Gaonker5209fe82016-04-19 10:09:53 -0700410 onos_ip = onos.ip()
ChetanGaonkerdbd4e4b2016-10-28 17:40:11 -0700411 onos_ips = [ onos_ip ]
412 num_onos_instances = args.onos_instances
413 if num_onos_instances > 1 and onos is not None:
414 onos_instances = []
415 onos_instances.append(onos)
416 for i in range(1, num_onos_instances):
417 name = '{}-{}'.format(Onos.NAME, i+1)
A R Karthick09dbc6d2016-11-22 10:37:42 -0800418 data_volume = '{}-data'.format(name) if args.shared_volume else None
A R Karthickec2db322016-11-17 15:06:01 -0800419 onos = Onos(name = name, image = Onos.IMAGE, tag = Onos.TAG, boot_delay = 60, cluster = cluster_mode,
420 data_volume = data_volume)
ChetanGaonkerdbd4e4b2016-10-28 17:40:11 -0700421 onos_instances.append(onos)
422 onos_ips.append(onos.ipaddr)
423 try:
424 for ip in onos_ips:
425 print('Installing cord tester ONOS app %s in ONOS instance %s' %(args.app,ip))
426 OnosCtrl.install_app(args.app, onos_ip = ip)
427 except: pass
A R Karthick1ef70552016-11-17 17:33:36 -0800428 if setup_cluster is True:
429 Onos.setup_cluster(onos_instances)
430 else:
431 print('ONOS instances already running. Skipping ONOS form cluster for %d instances' %num_onos_instances)
ChetanGaonkerdbd4e4b2016-10-28 17:40:11 -0700432 ctlr_addr = ','.join(onos_ips)
Chetan Gaonker5209fe82016-04-19 10:09:53 -0700433
A R Karthickeaf1c4e2016-07-19 12:22:35 -0700434 print('Onos IP %s, Test type %s' %(onos_ip, args.test_type))
A R Karthick07608ef2016-08-23 16:51:19 -0700435 if use_manifest or args.test_controller:
A R Karthickbd9b8a32016-07-21 09:56:45 -0700436 print('Installing ONOS cord apps')
A R Karthick07608ef2016-08-23 16:51:19 -0700437 try:
438 Onos.install_cord_apps(onos_ip = onos_ip)
439 except: pass
A R Karthickeaf1c4e2016-07-19 12:22:35 -0700440
ChetanGaonkerdbd4e4b2016-10-28 17:40:11 -0700441 if not cluster_mode:
442 print('Installing cord tester ONOS app %s' %args.app)
443 try:
444 for ip in onos_ips:
445 OnosCtrl.install_app(args.app, onos_ip = onos_ip)
446 except: pass
A R Karthickeaf1c4e2016-07-19 12:22:35 -0700447
448 if radius_ip is None:
A R Karthicka661b552016-05-25 10:18:50 -0700449 ##Start Radius container
A R Karthick07608ef2016-08-23 16:51:19 -0700450 radius = Radius(prefix = Container.IMAGE_PREFIX, update = update_map['radius'])
A R Karthicka661b552016-05-25 10:18:50 -0700451 radius_ip = radius.ip()
A.R Karthick95d044e2016-06-10 18:44:36 -0700452
A R Karthickeaf1c4e2016-07-19 12:22:35 -0700453 print('Radius server running with IP %s' %radius_ip)
A.R Karthick95d044e2016-06-10 18:44:36 -0700454
Chetan Gaonkerb84835f2016-04-19 15:12:10 -0700455 if args.quagga == True:
456 #Start quagga. Builds container if required
A R Karthick07608ef2016-08-23 16:51:19 -0700457 quagga = Quagga(prefix = Container.IMAGE_PREFIX, update = update_map['quagga'])
A R Karthick81acbff2016-06-17 14:45:16 -0700458
A R Karthick07608ef2016-08-23 16:51:19 -0700459 try:
460 maas_api_key = FabricMAAS.get_api_key()
461 except:
462 maas_api_key = 'UNKNOWN'
463
464 ssh_key_file = set_ssh_key_file(args.identity_file)
ChetanGaonkerdbd4e4b2016-10-28 17:40:11 -0700465 test_cnt_env = { 'ONOS_CONTROLLER_IP' : ctlr_addr,
Chetan Gaonkerc170f3f2016-04-19 17:24:45 -0700466 'ONOS_AAA_IP' : radius_ip if radius_ip is not None else '',
A R Karthick8d03cc52016-06-28 14:51:59 -0700467 'QUAGGA_IP': test_host,
A R Karthick81acbff2016-06-17 14:45:16 -0700468 'CORD_TEST_HOST' : test_host,
469 'CORD_TEST_PORT' : test_port,
A R Karthick4e0c0912016-08-17 16:57:42 -0700470 'ONOS_RESTART' : 0 if args.olt and args.test_controller else 1,
A R Karthick07608ef2016-08-23 16:51:19 -0700471 'MANIFEST': int(use_manifest),
472 'HEAD_NODE': head_node if head_node else CORD_TEST_HOST,
473 'MAAS_API_KEY': maas_api_key
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700474 }
A R Karthick07608ef2016-08-23 16:51:19 -0700475
476 if ssh_key_file:
477 test_cnt_env['SSH_KEY_FILE'] = ssh_key_file
478
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700479 if args.olt:
Chetan Gaonker7142a342016-04-07 14:53:12 -0700480 olt_conf_test_loc = os.path.join(CordTester.sandbox_setup, 'olt_config.json')
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700481 test_cnt_env['OLT_CONFIG'] = olt_conf_test_loc
482
A R Karthicka013a272016-08-16 16:40:19 -0700483 if args.num_containers > 1 and args.container:
484 print('Cannot specify number of containers with container option')
485 sys.exit(1)
486 if args.container:
487 args.keep = True
A R Karthick1f03e912016-05-18 11:39:22 -0700488 port_num = 0
489 num_tests = len(tests_parallel)
A R Karthick2b93d6a2016-09-06 15:19:09 -0700490 if num_tests > 0 and num_tests < args.num_containers:
491 tests_parallel *= args.num_containers/num_tests
492 num_tests = len(tests_parallel)
A R Karthick1f03e912016-05-18 11:39:22 -0700493 tests_per_container = max(1, num_tests/args.num_containers)
494 test_slice_start = 0
495 test_slice_end = test_slice_start + tests_per_container
496 num_test_containers = min(num_tests, args.num_containers)
497 if tests_parallel:
498 print('Running %s tests across %d containers in parallel' %(tests_parallel, num_test_containers))
499 for container in range(num_test_containers):
500 test_cnt = CordTester(tests_parallel[test_slice_start:test_slice_end],
501 instance = container, num_instances = num_test_containers,
ChetanGaonkerdbd4e4b2016-10-28 17:40:11 -0700502 ctlr_ip = ctlr_addr,
A R Karthick07608ef2016-08-23 16:51:19 -0700503 name = args.container,
504 image = nose_cnt['image'],
505 prefix = Container.IMAGE_PREFIX,
506 tag = nose_cnt['tag'],
A R Karthick1f03e912016-05-18 11:39:22 -0700507 env = test_cnt_env,
508 rm = False if args.keep else True,
509 update = update_map['test'])
510 test_slice_start = test_slice_end
511 test_slice_end = test_slice_start + tests_per_container
512 update_map['test'] = False
513 test_containers.append(test_cnt)
A R Karthicka013a272016-08-16 16:40:19 -0700514 if not test_cnt.create:
515 continue
516 if test_cnt.create and (args.start_switch or not args.olt):
A R Karthick07608ef2016-08-23 16:51:19 -0700517 if not args.no_switch:
518 test_cnt.start_switch()
A R Karthicka013a272016-08-16 16:40:19 -0700519 if test_cnt.create and test_cnt.olt:
A R Karthick1f03e912016-05-18 11:39:22 -0700520 _, port_num = test_cnt.setup_intfs(port_num = port_num)
521
A R Karthick9a5edc42016-08-24 19:10:22 -0700522 status = 0
523 if len(test_containers) > 1:
ChetanGaonkerdbd4e4b2016-10-28 17:40:11 -0700524 if True:
525 status = test_containers[0].run_tests()
526 else:
527 thread_pool = ThreadPool(len(test_containers), queue_size = 1, wait_timeout=1)
528 for test_cnt in test_containers:
529 thread_pool.addTask(test_cnt.run_tests)
530 thread_pool.cleanUpThreads()
A R Karthick9a5edc42016-08-24 19:10:22 -0700531 else:
A R Karthickcee37412016-08-29 10:10:56 -0700532 if test_containers:
533 status = test_containers[0].run_tests()
A R Karthick1f03e912016-05-18 11:39:22 -0700534
535 ##Run the linear tests
536 if tests_not_parallel:
537 test_cnt = CordTester(tests_not_parallel,
ChetanGaonkerdbd4e4b2016-10-28 17:40:11 -0700538 ctlr_ip = ctlr_addr,
A R Karthick07608ef2016-08-23 16:51:19 -0700539 name = args.container,
540 image = nose_cnt['image'],
541 prefix = Container.IMAGE_PREFIX,
542 tag = nose_cnt['tag'],
A R Karthick1f03e912016-05-18 11:39:22 -0700543 env = test_cnt_env,
544 rm = False if args.keep else True,
545 update = update_map['test'])
A R Karthicka013a272016-08-16 16:40:19 -0700546 if test_cnt.create and (args.start_switch or not args.olt):
A R Karthick36cfcef2016-08-18 15:20:07 -0700547 #For non parallel tests, we just restart the switch also for OLT's
548 CordTester.switch_on_olt = False
A R Karthick07608ef2016-08-23 16:51:19 -0700549 if not args.no_switch:
550 test_cnt.start_switch()
A R Karthicka013a272016-08-16 16:40:19 -0700551 if test_cnt.create and test_cnt.olt:
A R Karthick1f03e912016-05-18 11:39:22 -0700552 test_cnt.setup_intfs(port_num = port_num)
ChetanGaonkerdbd4e4b2016-10-28 17:40:11 -0700553 test_cnt.run_tests()
A R Karthick1f03e912016-05-18 11:39:22 -0700554
A R Karthick81acbff2016-06-17 14:45:16 -0700555 if test_server:
556 cord_test_server_stop(test_server)
557
A R Karthick9a5edc42016-08-24 19:10:22 -0700558 return status
559
A R Karthick81acbff2016-06-17 14:45:16 -0700560##Starts onos/radius/quagga containers as appropriate
561def setupCordTester(args):
562 onos_cnt = {'tag':'latest'}
A R Karthick07608ef2016-08-23 16:51:19 -0700563 nose_cnt = {'image': CordTester.IMAGE, 'tag': 'candidate'}
A R Karthick92a0e5a2016-06-22 17:11:05 -0700564 update_map = { 'quagga' : False, 'radius' : False, 'test': False }
A R Karthick81acbff2016-06-17 14:45:16 -0700565 update_map[args.update.lower()] = True
566
567 if args.update.lower() == 'all':
568 for c in update_map.keys():
569 update_map[c] = True
570
571 onos_ip = None
572 radius_ip = None
A R Karthickd44cea12016-07-20 12:16:41 -0700573 onos_cord_loc = args.onos_cord
574 if onos_cord_loc:
575 if onos_cord_loc.find(os.path.sep) < 0:
576 onos_cord_loc = os.path.join(os.getenv('HOME'), onos_cord_loc)
577 if not os.access(onos_cord_loc, os.F_OK):
578 print('ONOS cord config location %s is not accessible' %onos_cord_loc)
579 sys.exit(1)
580 #Disable test container provisioning on the ONOS compute node
581 args.dont_provision = True
A R Karthick81acbff2016-06-17 14:45:16 -0700582
A R Karthick07608ef2016-08-23 16:51:19 -0700583 head_node = platform.node()
584 use_manifest = False
585 if args.manifest:
586 if os.access(args.manifest, os.F_OK):
587 ##copy it to setup directory
588 dest = os.path.join(CordTester.tester_base, 'manifest.json')
589 if os.path.abspath(args.manifest) != dest:
590 try:
591 shutil.copy(args.manifest, dest)
592 except: pass
593 test_manifest = TestManifest(dest)
594 onos_ip = test_manifest.onos_ip
595 radius_ip = test_manifest.radius_ip
596 head_node = test_manifest.head_node
597 use_manifest = True
598
A R Karthick81acbff2016-06-17 14:45:16 -0700599 ##If onos/radius was already started
600 if args.test_controller:
601 ips = args.test_controller.split('/')
602 onos_ip = ips[0]
603 if len(ips) > 1:
604 radius_ip = ips[1]
605 else:
606 radius_ip = None
607
A R Karthickd44cea12016-07-20 12:16:41 -0700608 onos_cord = None
609 if onos_cord_loc:
610 if not args.test_controller:
611 ##Unexpected case. Specify the external controller ip when running on cord node
612 print('Specify ONOS ip using \"-e\" option when running the cord-tester on cord node')
613 sys.exit(1)
A R Karthickbd9b8a32016-07-21 09:56:45 -0700614 onos_cord = OnosCord(onos_ip, onos_cord_loc)
A R Karthickd44cea12016-07-20 12:16:41 -0700615
A R Karthick07608ef2016-08-23 16:51:19 -0700616 Container.IMAGE_PREFIX = args.prefix
A R Karthick81acbff2016-06-17 14:45:16 -0700617 #don't spawn onos if the user had started it externally
A R Karthick07608ef2016-08-23 16:51:19 -0700618 image_names = args.onos.rsplit(':', 1)
619 onos_cnt['image'] = image_names[0]
620 if len(image_names) > 1:
621 if image_names[1].find('/') < 0:
622 onos_cnt['tag'] = image_names[1]
623 else:
624 #tag cannot have slashes
625 onos_cnt['image'] = args.onos
A R Karthick81acbff2016-06-17 14:45:16 -0700626
A R Karthick07608ef2016-08-23 16:51:19 -0700627 Onos.IMAGE = onos_cnt['image']
628 Onos.PREFIX = args.prefix
629 Onos.TAG = onos_cnt['tag']
A R Karthick2b93d6a2016-09-06 15:19:09 -0700630 cluster_mode = True if args.onos_instances > 1 else False
A R Karthick51e6fd82016-11-22 14:39:19 -0800631 existing_list = [ c['Names'][0][1:] for c in Container.dckr.containers() if c['Image'] == args.onos ]
632 setup_cluster = False if len(existing_list) == args.onos_instances else True
A R Karthick2b93d6a2016-09-06 15:19:09 -0700633 onos = None
A R Karthick81acbff2016-06-17 14:45:16 -0700634 if onos_ip is None:
A R Karthick09dbc6d2016-11-22 10:37:42 -0800635 data_volume = '{}-data'.format(Onos.NAME) if args.shared_volume else None
A R Karthickec2db322016-11-17 15:06:01 -0800636 onos = Onos(image = Onos.IMAGE, tag = Onos.TAG, boot_delay = 60, cluster = cluster_mode,
637 data_volume = data_volume)
A R Karthick81acbff2016-06-17 14:45:16 -0700638 onos_ip = onos.ip()
639
A R Karthick2b93d6a2016-09-06 15:19:09 -0700640 num_onos_instances = args.onos_instances
641 onos_ips = [ onos_ip ]
642 if num_onos_instances > 1 and onos is not None:
643 onos_instances = []
644 onos_instances.append(onos)
645 for i in range(1, num_onos_instances):
646 name = '{}-{}'.format(Onos.NAME, i+1)
A R Karthick09dbc6d2016-11-22 10:37:42 -0800647 data_volume = '{}-data'.format(name) if args.shared_volume else None
A R Karthickec2db322016-11-17 15:06:01 -0800648 onos = Onos(name = name, image = Onos.IMAGE, tag = Onos.TAG, boot_delay = 60, cluster = cluster_mode,
649 data_volume = data_volume)
A R Karthick2b93d6a2016-09-06 15:19:09 -0700650 onos_instances.append(onos)
651 onos_ips.append(onos.ipaddr)
A R Karthick51e6fd82016-11-22 14:39:19 -0800652 if setup_cluster is True:
653 Onos.setup_cluster(onos_instances)
A R Karthick2b93d6a2016-09-06 15:19:09 -0700654
655 ctlr_addr = ','.join(onos_ips)
656 print('Onos IP %s' %ctlr_addr)
A R Karthick07608ef2016-08-23 16:51:19 -0700657 if use_manifest or args.test_controller:
A R Karthickbd9b8a32016-07-21 09:56:45 -0700658 print('Installing ONOS cord apps')
A R Karthick07608ef2016-08-23 16:51:19 -0700659 try:
660 Onos.install_cord_apps(onos_ip = onos_ip)
661 except: pass
A R Karthickbd9b8a32016-07-21 09:56:45 -0700662
A R Karthickedab01c2016-09-08 14:05:44 -0700663 print('Installing cord tester ONOS app %s' %args.app)
A R Karthick07608ef2016-08-23 16:51:19 -0700664 try:
A R Karthick2b93d6a2016-09-06 15:19:09 -0700665 for ip in onos_ips:
666 OnosCtrl.install_app(args.app, onos_ip = ip)
A R Karthick07608ef2016-08-23 16:51:19 -0700667 except: pass
A R Karthickeaf1c4e2016-07-19 12:22:35 -0700668
A R Karthick81acbff2016-06-17 14:45:16 -0700669 ##Start Radius container if not started
670 if radius_ip is None:
A R Karthick07608ef2016-08-23 16:51:19 -0700671 radius = Radius(prefix = Container.IMAGE_PREFIX, update = update_map['radius'])
A R Karthick81acbff2016-06-17 14:45:16 -0700672 radius_ip = radius.ip()
673
674 print('Radius server running with IP %s' %radius_ip)
A R Karthick81acbff2016-06-17 14:45:16 -0700675
676 if args.quagga == True:
677 #Start quagga. Builds container if required
A R Karthick07608ef2016-08-23 16:51:19 -0700678 quagga = Quagga(prefix = Container.IMAGE_PREFIX, update = update_map['quagga'])
A R Karthick8d03cc52016-06-28 14:51:59 -0700679 print('Quagga started')
A R Karthick81acbff2016-06-17 14:45:16 -0700680
A R Karthick81acbff2016-06-17 14:45:16 -0700681 params = args.server.split(':')
682 ip = params[0]
683 port = CORD_TEST_PORT
684 if len(params) > 1:
685 port = int(params[1])
A R Karthick92a0e5a2016-06-22 17:11:05 -0700686
A R Karthick07608ef2016-08-23 16:51:19 -0700687 try:
688 maas_api_key = FabricMAAS.get_api_key()
689 except:
690 maas_api_key = 'UNKNOWN'
691
692 ssh_key_file = set_ssh_key_file(args.identity_file)
693
A R Karthick92a0e5a2016-06-22 17:11:05 -0700694 #provision the test container
695 if not args.dont_provision:
A R Karthick2b93d6a2016-09-06 15:19:09 -0700696 test_cnt_env = { 'ONOS_CONTROLLER_IP' : ctlr_addr,
A R Karthick92a0e5a2016-06-22 17:11:05 -0700697 'ONOS_AAA_IP' : radius_ip,
A R Karthick8d03cc52016-06-28 14:51:59 -0700698 'QUAGGA_IP': ip,
A R Karthick92a0e5a2016-06-22 17:11:05 -0700699 'CORD_TEST_HOST' : ip,
700 'CORD_TEST_PORT' : port,
A R Karthick4e0c0912016-08-17 16:57:42 -0700701 'ONOS_RESTART' : 0 if args.olt and args.test_controller else 1,
A R Karthick07608ef2016-08-23 16:51:19 -0700702 'MANIFEST': int(use_manifest),
703 'HEAD_NODE': head_node if head_node else CORD_TEST_HOST,
704 'MAAS_API_KEY': maas_api_key
A R Karthick92a0e5a2016-06-22 17:11:05 -0700705 }
A R Karthick07608ef2016-08-23 16:51:19 -0700706
707 if ssh_key_file:
708 test_cnt_env['SSH_KEY_FILE'] = ssh_key_file
A R Karthick92a0e5a2016-06-22 17:11:05 -0700709 if args.olt:
710 olt_conf_test_loc = os.path.join(CordTester.sandbox_setup, 'olt_config.json')
711 test_cnt_env['OLT_CONFIG'] = olt_conf_test_loc
712
713 test_cnt = CordTester((),
A R Karthick2b93d6a2016-09-06 15:19:09 -0700714 ctlr_ip = ctlr_addr,
A R Karthick92a0e5a2016-06-22 17:11:05 -0700715 image = nose_cnt['image'],
A R Karthick07608ef2016-08-23 16:51:19 -0700716 prefix = Container.IMAGE_PREFIX,
A R Karthick92a0e5a2016-06-22 17:11:05 -0700717 tag = nose_cnt['tag'],
718 env = test_cnt_env,
719 rm = False,
720 update = update_map['test'])
721
722 if args.start_switch or not args.olt:
723 test_cnt.start_switch()
724 if test_cnt.olt:
725 test_cnt.setup_intfs(port_num = 0)
726 print('Test container %s started and provisioned to run tests using nosetests' %(test_cnt.name))
727
728 #Finally start the test server and daemonize
A R Karthick14118c62016-07-27 14:54:04 -0700729 try:
A R Karthickbd82f362016-11-10 15:08:52 -0800730 cord_test_server_start(daemonize = not args.foreground, cord_test_host = ip, cord_test_port = port,
731 onos_cord = onos_cord, foreground = args.foreground)
A R Karthick14118c62016-07-27 14:54:04 -0700732 except socket.error, e:
733 #the test agent address could be remote or already running. Exit gracefully
734 sys.exit(0)
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700735
A R Karthick9a5edc42016-08-24 19:10:22 -0700736 return 0
737
Chetan Gaonker503032a2016-05-12 12:06:29 -0700738def cleanupTests(args):
A R Karthick2b93d6a2016-09-06 15:19:09 -0700739 image_name = args.onos
A R Karthick07608ef2016-08-23 16:51:19 -0700740 prefix = args.prefix
741 if prefix:
742 prefix += '/'
743 test_container = '{}{}:candidate'.format(prefix, CordTester.IMAGE)
Chetan Gaonker503032a2016-05-12 12:06:29 -0700744 print('Cleaning up Test containers ...')
745 Container.cleanup(test_container)
A R Karthickb50f5592016-07-26 12:19:29 -0700746 if args.olt:
747 print('Cleaning up test container OLT configuration')
748 CordTester.cleanup_intfs()
A R Karthick2b93d6a2016-09-06 15:19:09 -0700749
750 onos_list = [ c['Names'][0][1:] for c in Container.dckr.containers() if c['Image'] == image_name ]
751 if len(onos_list) > 1:
752 for onos in onos_list:
753 Container.dckr.kill(onos)
754 Container.dckr.remove_container(onos, force=True)
A R Karthickec2db322016-11-17 15:06:01 -0800755 for index in range(len(onos_list)):
756 volume = '{}-data'.format(Onos.NAME) if index == 0 else '{}-{}-data'.format(Onos.NAME, index+1)
757 Onos.remove_data_map(volume, Onos.guest_data_dir)
A R Karthick9d48c652016-09-15 09:16:36 -0700758 Onos.cleanup_runtime()
A R Karthickec2db322016-11-17 15:06:01 -0800759
A.R Karthick842f0122016-09-28 14:48:47 -0700760 if args.xos:
761 ##cleanup XOS images
762 xos_images = ( '{}:{}'.format(XosServer.IMAGE,XosServer.TAG),
763 '{}:{}'.format(XosSynchronizerOpenstack.IMAGE,
764 XosSynchronizerOpenstack.TAG),
765 '{}:{}'.format(XosSynchronizerOnboarding.IMAGE,
766 XosSynchronizerOnboarding.TAG),
767 '{}:{}'.format(XosSynchronizerOpenvpn.IMAGE,
768 XosSynchronizerOpenvpn.TAG),
769 '{}:{}'.format(XosPostgresql.IMAGE,
770 XosPostgresql.TAG),
771 '{}:{}'.format(XosSyndicateMs.IMAGE,
772 XosSyndicateMs.TAG),
773 )
774 for img in xos_images:
775 print('Cleaning up XOS image: %s' %img)
776 Container.cleanup(img)
777
A R Karthicke99ab5c2016-09-30 13:59:57 -0700778 server_params = args.server.split(':')
779 server_host = server_params[0]
780 server_port = CORD_TEST_PORT
781 if len(server_params) > 1:
782 server_port = int(server_params[1])
783 cord_test_server_shutdown(server_host, server_port)
A R Karthick9a5edc42016-08-24 19:10:22 -0700784 return 0
Chetan Gaonker503032a2016-05-12 12:06:29 -0700785
786def listTests(args):
787 if args.test == 'all':
788 tests = CordTester.ALL_TESTS
789 else:
A R Karthickacae3b42016-05-12 15:27:24 -0700790 tests = args.test.split('-')
Chetan Gaonker503032a2016-05-12 12:06:29 -0700791 CordTester.list_tests(tests)
A R Karthickcee37412016-08-29 10:10:56 -0700792 return 0
ChetanGaonkereadad482016-08-26 01:21:47 -0700793
794def getMetrics(args):
A R Karthickcee37412016-08-29 10:10:56 -0700795 try:
796 detail = c.inspect_container(args.container)
797 except:
798 print('Unknown container %s' %args.container)
799 return 0
800 user_hz = os.sysconf(os.sysconf_names['SC_CLK_TCK'])
ChetanGaonkereadad482016-08-26 01:21:47 -0700801 state = detail["State"]
802 if bool(state["Paused"]):
803 print("Container is in Paused State")
804 elif bool(state["Running"]):
805 print("Container is in Running State")
806 elif int(state["ExitCode"]) == 0:
807 print("Container is in Stopped State")
808 else:
809 print("Container is in Crashed State")
810
A R Karthickcee37412016-08-29 10:10:56 -0700811 print("Ip Address of the container: " +detail['NetworkSettings']['IPAddress'])
ChetanGaonkereadad482016-08-26 01:21:47 -0700812
813 if bool(detail["State"]["Running"]):
814 container_id = detail['Id']
815 cpu_usage = {}
A R Karthickcee37412016-08-29 10:10:56 -0700816 cur_usage = 0
817 last_usage = 0
818 for i in range(2):
819 with open('/sys/fs/cgroup/cpuacct/docker/' + container_id + '/cpuacct.stat', 'r') as f:
820 for line in f:
821 m = re.search(r"(system|user)\s+(\d+)", line)
822 if m:
823 cpu_usage[m.group(1)] = int(m.group(2))
824 cpu = cpu_usage["system"] + cpu_usage["user"]
825 last_usage = cur_usage
826 cur_usage = cpu
827 time.sleep(1)
828 cpu_percent = (cur_usage - last_usage)*100.0/user_hz
829 print("CPU Usage: %.2f %%" %(cpu_percent))
ChetanGaonkereadad482016-08-26 01:21:47 -0700830 else:
831 print(0)
832
833 if bool(detail["State"]["Running"]):
834 container_id = detail['Id']
A R Karthickcee37412016-08-29 10:10:56 -0700835 print("Docker Port Info:")
ChetanGaonkereadad482016-08-26 01:21:47 -0700836 cmd = "sudo docker port {}".format(container_id)
837 os.system(cmd)
838
839 if bool(detail["State"]["Running"]):
840 container_id = detail['Id']
841 with open('/sys/fs/cgroup/memory/docker/' + container_id + '/memory.stat', 'r') as f:
842 for line in f:
843 m = re.search(r"total_rss\s+(\d+)", line)
844 if m:
A R Karthickcee37412016-08-29 10:10:56 -0700845 mem = int(m.group(1))
846 print("Memory: %s KB "%(mem/1024.0))
ChetanGaonkereadad482016-08-26 01:21:47 -0700847 o = re.search(r"usage\s+(\d+)", line)
848 if o:
A R Karthickcee37412016-08-29 10:10:56 -0700849 print("Usage: %s "%(o.group(1)))
ChetanGaonkereadad482016-08-26 01:21:47 -0700850 p = re.search(r"max_usage\s+(\d+)", line)
851 if p:
A R Karthickcee37412016-08-29 10:10:56 -0700852 print("Max Usage: %s "%(p.group(1)))
ChetanGaonkereadad482016-08-26 01:21:47 -0700853
854 if bool(detail["State"]["Running"]):
855 container_id = detail['Id']
856 with open('/sys/fs/cgroup/cpuacct/docker/' + container_id + '/cpuacct.stat', 'r') as f:
857 for line in f:
858 m = re.search(r"user\s+(\d+)", line)
859 if m:
A R Karthickcee37412016-08-29 10:10:56 -0700860 user_ticks = int(m.group(1))
861 print("Time spent by running processes: %.2f ms"%(user_ticks*1000.0/user_hz))
862 print("List Networks:")
ChetanGaonkereadad482016-08-26 01:21:47 -0700863 cmd = "docker network ls"
864 os.system(cmd)
A R Karthick9a5edc42016-08-24 19:10:22 -0700865 return 0
Chetan Gaonker503032a2016-05-12 12:06:29 -0700866
867def buildImages(args):
A R Karthick07608ef2016-08-23 16:51:19 -0700868 tag = 'candidate'
869 prefix = args.prefix
870 if prefix:
871 prefix += '/'
Chetan Gaonker503032a2016-05-12 12:06:29 -0700872 if args.image == 'all' or args.image == 'quagga':
A R Karthick07608ef2016-08-23 16:51:19 -0700873 image_name = '{}{}:{}'.format(prefix, Quagga.IMAGE, tag)
874 Quagga.build_image(image_name)
A.R Karthick95d044e2016-06-10 18:44:36 -0700875
Chetan Gaonker503032a2016-05-12 12:06:29 -0700876 if args.image == 'all' or args.image == 'radius':
A R Karthick07608ef2016-08-23 16:51:19 -0700877 image_name = '{}{}:{}'.format(prefix, Radius.IMAGE, tag)
878 Radius.build_image(image_name)
Chetan Gaonker503032a2016-05-12 12:06:29 -0700879
880 if args.image == 'all' or args.image == 'test':
A R Karthick07608ef2016-08-23 16:51:19 -0700881 image_name = '{}{}:{}'.format(prefix, CordTester.IMAGE, tag)
882 CordTester.build_image(image_name)
Chetan Gaonker503032a2016-05-12 12:06:29 -0700883
A R Karthick9a5edc42016-08-24 19:10:22 -0700884 return 0
885
A R Karthickbec27762016-07-28 10:59:34 -0700886def startImages(args):
A R Karthickbec27762016-07-28 10:59:34 -0700887 ##starts the latest ONOS image
A R Karthick07608ef2016-08-23 16:51:19 -0700888 onos_cnt = {'tag': 'latest'}
889 image_names = args.onos.rsplit(':', 1)
890 onos_cnt['image'] = image_names[0]
891 if len(image_names) > 1:
892 if image_names[1].find('/') < 0:
893 onos_cnt['tag'] = image_names[1]
894 else:
895 #tag cannot have slashes
896 onos_cnt['image'] = args.onos
897
A R Karthickbec27762016-07-28 10:59:34 -0700898 if args.image == 'all' or args.image == 'onos':
A R Karthick07608ef2016-08-23 16:51:19 -0700899 onos = Onos(image = onos_cnt['image'], tag = onos_cnt['tag'])
A R Karthickbec27762016-07-28 10:59:34 -0700900 print('ONOS started with ip %s' %(onos.ip()))
901
902 if args.image == 'all' or args.image == 'quagga':
A R Karthick07608ef2016-08-23 16:51:19 -0700903 quagga = Quagga(prefix = args.prefix)
A R Karthickbec27762016-07-28 10:59:34 -0700904 print('Quagga started with ip %s' %(quagga.ip()))
905
906 if args.image == 'all' or args.image == 'radius':
A R Karthick07608ef2016-08-23 16:51:19 -0700907 radius = Radius(prefix = args.prefix)
A R Karthickbec27762016-07-28 10:59:34 -0700908 print('Radius started with ip %s' %(radius.ip()))
909
A R Karthick9a5edc42016-08-24 19:10:22 -0700910 return 0
911
A R Karthickea8bfce2016-10-13 16:32:07 -0700912def xosCommand(args):
913 update = False
914 profile = args.profile
915 if args.command == 'update':
916 update = True
917 xos = XosServiceProfile(profile = profile, update = update)
918 if args.command == 'build':
919 xos.build_images(force = True)
920 if args.command == 'start':
921 xos.start_services()
922 if args.command == 'stop':
923 xos.stop_services(rm = True)
ChetanGaonker2c0e9bb2016-09-21 13:38:37 -0700924 return 0
925
Chetan Gaonker93e302d2016-04-05 10:51:07 -0700926if __name__ == '__main__':
Chetan Gaonker678743f2016-04-26 09:54:31 -0700927 parser = ArgumentParser(description='Cord Tester')
Chetan Gaonker503032a2016-05-12 12:06:29 -0700928 subparser = parser.add_subparsers()
929 parser_run = subparser.add_parser('run', help='Run cord tester')
930 parser_run.add_argument('-t', '--test-type', default=test_type_default, help='Specify test type or test case to run')
931 parser_run.add_argument('-o', '--onos', default=onos_image_default, type=str, help='ONOS container image')
Chetan Gaonker503032a2016-05-12 12:06:29 -0700932 parser_run.add_argument('-q', '--quagga',action='store_true',help='Provision quagga container for vrouter')
933 parser_run.add_argument('-a', '--app', default=onos_app_file, type=str, help='Cord ONOS app filename')
A R Karthick07608ef2016-08-23 16:51:19 -0700934 parser_run.add_argument('-l', '--olt', action='store_true', help='Use OLT config')
Chetan Gaonker503032a2016-05-12 12:06:29 -0700935 parser_run.add_argument('-e', '--test-controller', default='', type=str, help='External test controller ip for Onos and/or radius server. '
Chetan Gaonker5209fe82016-04-19 10:09:53 -0700936 'Eg: 10.0.0.2/10.0.0.3 to specify ONOS and Radius ip to connect')
A R Karthick81acbff2016-06-17 14:45:16 -0700937 parser_run.add_argument('-r', '--server', default=cord_test_server_address, type=str,
938 help='ip:port address to connect for cord test server for container requests')
Chetan Gaonker503032a2016-05-12 12:06:29 -0700939 parser_run.add_argument('-k', '--keep', action='store_true', help='Keep test container after tests')
940 parser_run.add_argument('-s', '--start-switch', action='store_true', help='Start OVS when running under OLT config')
941 parser_run.add_argument('-u', '--update', default='none', choices=['test','quagga','radius', 'all'], type=str, help='Update cord tester container images. '
942 'Eg: --update=quagga to rebuild quagga image.'
943 ' --update=radius to rebuild radius server image.'
944 ' --update=test to rebuild cord test image.(Default)'
945 ' --update=all to rebuild all cord tester images.')
A R Karthick1f03e912016-05-18 11:39:22 -0700946 parser_run.add_argument('-n', '--num-containers', default=1, type=int,
947 help='Specify number of test containers to spawn for tests')
A R Karthicka013a272016-08-16 16:40:19 -0700948 parser_run.add_argument('-c', '--container', default='', type=str, help='Test container name for running tests')
A R Karthick07608ef2016-08-23 16:51:19 -0700949 parser_run.add_argument('-m', '--manifest', default='', type=str, help='Provide test configuration manifest')
950 parser_run.add_argument('-p', '--prefix', default='', type=str, help='Provide container image prefix')
951 parser_run.add_argument('-d', '--no-switch', action='store_true', help='Dont start test switch.')
952 parser_run.add_argument('-i', '--identity-file', default=identity_file_default,
953 type=str, help='ssh identity file to access compute nodes from test container')
ChetanGaonkerdbd4e4b2016-10-28 17:40:11 -0700954 parser_run.add_argument('-j', '--onos-instances', default=1, type=int,
955 help='Specify number to test onos instances to form cluster')
A R Karthick09dbc6d2016-11-22 10:37:42 -0800956 parser_run.add_argument('-v', '--shared-volume', action='store_true', help='Start ONOS cluster instances with shared volume')
Chetan Gaonker503032a2016-05-12 12:06:29 -0700957 parser_run.set_defaults(func=runTest)
958
A R Karthick81acbff2016-06-17 14:45:16 -0700959
960 parser_setup = subparser.add_parser('setup', help='Setup cord tester environment')
961 parser_setup.add_argument('-o', '--onos', default=onos_image_default, type=str, help='ONOS container image')
962 parser_setup.add_argument('-r', '--server', default=cord_test_server_address, type=str,
963 help='ip:port address for cord test server to listen for container restart requests')
964 parser_setup.add_argument('-q', '--quagga',action='store_true',help='Provision quagga container for vrouter')
965 parser_setup.add_argument('-a', '--app', default=onos_app_file, type=str, help='Cord ONOS app filename')
966 parser_setup.add_argument('-e', '--test-controller', default='', type=str, help='External test controller ip for Onos and/or radius server. '
967 'Eg: 10.0.0.2/10.0.0.3 to specify ONOS and Radius ip to connect')
968 parser_setup.add_argument('-u', '--update', default='none', choices=['quagga','radius', 'all'], type=str, help='Update cord tester container images. '
969 'Eg: --update=quagga to rebuild quagga image.'
970 ' --update=radius to rebuild radius server image.'
971 ' --update=all to rebuild all cord tester images.')
A R Karthick92a0e5a2016-06-22 17:11:05 -0700972 parser_setup.add_argument('-d', '--dont-provision', action='store_true', help='Dont start test container.')
A R Karthick07608ef2016-08-23 16:51:19 -0700973 parser_setup.add_argument('-l', '--olt', action='store_true', help='Use OLT config')
A R Karthick92a0e5a2016-06-22 17:11:05 -0700974 parser_setup.add_argument('-s', '--start-switch', action='store_true', help='Start OVS when running under OLT config')
A R Karthickd44cea12016-07-20 12:16:41 -0700975 parser_setup.add_argument('-c', '--onos-cord', default='', type=str,
976 help='Specify cord location for ONOS cord when running on podd')
A R Karthick07608ef2016-08-23 16:51:19 -0700977 parser_setup.add_argument('-m', '--manifest', default='', type=str, help='Provide test configuration manifest')
978 parser_setup.add_argument('-p', '--prefix', default='', type=str, help='Provide container image prefix')
979 parser_setup.add_argument('-i', '--identity-file', default=identity_file_default,
980 type=str, help='ssh identity file to access compute nodes from test container')
A R Karthick2b93d6a2016-09-06 15:19:09 -0700981 parser_setup.add_argument('-n', '--onos-instances', default=1, type=int,
A R Karthickbd82f362016-11-10 15:08:52 -0800982 help='Specify number of test onos instances to spawn')
A R Karthick09dbc6d2016-11-22 10:37:42 -0800983 parser_setup.add_argument('-v', '--shared-volume', action='store_true',
984 help='Start ONOS cluster instances with shared volume')
A R Karthickbd82f362016-11-10 15:08:52 -0800985 parser_setup.add_argument('-f', '--foreground', action='store_true', help='Run in foreground')
A R Karthick81acbff2016-06-17 14:45:16 -0700986 parser_setup.set_defaults(func=setupCordTester)
987
ChetanGaonker2c0e9bb2016-09-21 13:38:37 -0700988 parser_xos = subparser.add_parser('xos', help='Building xos into cord tester environment')
A R Karthickea8bfce2016-10-13 16:32:07 -0700989 parser_xos.add_argument('command', choices=['build', 'update', 'start', 'stop'])
990 parser_xos.add_argument('-p', '--profile', default='cord-pod', type=str, help='Provide service profile')
991 parser_xos.set_defaults(func=xosCommand)
ChetanGaonker2c0e9bb2016-09-21 13:38:37 -0700992
Chetan Gaonker503032a2016-05-12 12:06:29 -0700993 parser_list = subparser.add_parser('list', help='List test cases')
994 parser_list.add_argument('-t', '--test', default='all', help='Specify test type to list test cases. '
995 'Eg: -t tls to list tls test cases.'
996 ' -t tls-dhcp-vrouter to list tls,dhcp and vrouter test cases.'
997 ' -t all to list all test cases.')
998 parser_list.set_defaults(func=listTests)
999
1000 parser_build = subparser.add_parser('build', help='Build cord test container images')
ChetanGaonker2c0e9bb2016-09-21 13:38:37 -07001001 parser_build.add_argument('image', choices=['quagga', 'radius', 'test','all'])
A.R Karthick842f0122016-09-28 14:48:47 -07001002 parser_build.add_argument('-p', '--prefix', default='', type=str, help='Provide container image prefix')
Chetan Gaonker503032a2016-05-12 12:06:29 -07001003 parser_build.set_defaults(func=buildImages)
1004
ChetanGaonkereadad482016-08-26 01:21:47 -07001005 parser_metrics = subparser.add_parser('metrics', help='Info of container')
1006 parser_metrics.add_argument("container", help="Container name")
1007 parser_metrics.set_defaults(func=getMetrics)
1008
A R Karthickbec27762016-07-28 10:59:34 -07001009 parser_start = subparser.add_parser('start', help='Start cord tester containers')
A R Karthick07608ef2016-08-23 16:51:19 -07001010 parser_start.add_argument('-p', '--prefix', default='', type=str, help='Provide container image prefix')
1011 parser_start.add_argument('-o', '--onos', default=onos_image_default, type=str, help='ONOS container image')
A R Karthickbec27762016-07-28 10:59:34 -07001012 parser_start.add_argument('image', choices=['onos', 'quagga', 'radius', 'all'])
1013 parser_start.set_defaults(func=startImages)
1014
Chetan Gaonker503032a2016-05-12 12:06:29 -07001015 parser_cleanup = subparser.add_parser('cleanup', help='Cleanup test containers')
A R Karthick07608ef2016-08-23 16:51:19 -07001016 parser_cleanup.add_argument('-p', '--prefix', default='', type=str, help='Provide container image prefix')
1017 parser_cleanup.add_argument('-l', '--olt', action = 'store_true', help = 'Cleanup OLT config')
A R Karthick2b93d6a2016-09-06 15:19:09 -07001018 parser_cleanup.add_argument('-o', '--onos', default=onos_image_default, type=str,
1019 help='ONOS container image to cleanup')
A.R Karthick842f0122016-09-28 14:48:47 -07001020 parser_cleanup.add_argument('-x', '--xos', action='store_true',
1021 help='Cleanup XOS containers')
A R Karthicke99ab5c2016-09-30 13:59:57 -07001022 parser_cleanup.add_argument('-r', '--server', default=cord_test_server_address, type=str,
1023 help='ip:port address for cord test server to cleanup')
Chetan Gaonker503032a2016-05-12 12:06:29 -07001024 parser_cleanup.set_defaults(func=cleanupTests)
1025
ChetanGaonkereadad482016-08-26 01:21:47 -07001026 c = Client(**(kwargs_from_env()))
1027
Chetan Gaonker93e302d2016-04-05 10:51:07 -07001028 args = parser.parse_args()
A R Karthick9a5edc42016-08-24 19:10:22 -07001029 res = args.func(args)
1030 sys.exit(res)