Chetan Gaonker | 93e302d | 2016-04-05 10:51:07 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | from argparse import ArgumentParser |
| 3 | import os,sys,time |
| 4 | import io |
| 5 | import yaml |
| 6 | from pyroute2 import IPRoute |
| 7 | from itertools import chain |
| 8 | from nsenter import Namespace |
| 9 | from docker import Client |
| 10 | from shutil import copy |
Chetan Gaonker | 7142a34 | 2016-04-07 14:53:12 -0700 | [diff] [blame^] | 11 | utils_dir = os.path.join( os.path.dirname(os.path.realpath(sys.argv[0])), '../utils') |
| 12 | sys.path.append(utils_dir) |
Chetan Gaonker | 93e302d | 2016-04-05 10:51:07 -0700 | [diff] [blame] | 13 | from OnosCtrl import OnosCtrl |
| 14 | |
| 15 | class docker_netns(object): |
| 16 | |
| 17 | dckr = Client() |
| 18 | def __init__(self, name): |
| 19 | pid = int(self.dckr.inspect_container(name)['State']['Pid']) |
| 20 | if pid == 0: |
| 21 | raise Exception('no container named {0}'.format(name)) |
| 22 | self.pid = pid |
| 23 | |
| 24 | def __enter__(self): |
| 25 | pid = self.pid |
| 26 | if not os.path.exists('/var/run/netns'): |
| 27 | os.mkdir('/var/run/netns') |
| 28 | os.symlink('/proc/{0}/ns/net'.format(pid), '/var/run/netns/{0}'.format(pid)) |
| 29 | return str(pid) |
| 30 | |
| 31 | def __exit__(self, type, value, traceback): |
| 32 | pid = self.pid |
| 33 | os.unlink('/var/run/netns/{0}'.format(pid)) |
| 34 | |
| 35 | flatten = lambda l: chain.from_iterable(l) |
| 36 | |
| 37 | class Container(object): |
| 38 | dckr = Client() |
| 39 | def __init__(self, name, image, tag = 'latest', command = 'bash', quagga_config = None): |
| 40 | self.name = name |
| 41 | self.image = image |
| 42 | self.tag = tag |
| 43 | self.image_name = image + ':' + tag |
| 44 | self.id = None |
| 45 | self.command = command |
| 46 | if quagga_config is not None: |
| 47 | self.bridge = quagga_config['bridge'] |
| 48 | self.ipaddress = quagga_config['ip'] |
| 49 | self.mask = quagga_config['mask'] |
| 50 | else: |
| 51 | self.bridge = None |
| 52 | self.ipaddress = None |
| 53 | self.mask = None |
| 54 | |
| 55 | @classmethod |
| 56 | def build_image(cls, dockerfile, tag, force=True, nocache=False): |
| 57 | f = io.BytesIO(dockerfile.encode('utf-8')) |
| 58 | if force or not cls.image_exists(tag): |
| 59 | print('Build {0}...'.format(tag)) |
| 60 | for line in cls.dckr.build(fileobj=f, rm=True, tag=tag, decode=True, nocache=nocache): |
| 61 | if 'stream' in line: |
| 62 | print(line['stream'].strip()) |
| 63 | |
| 64 | @classmethod |
| 65 | def image_exists(cls, name): |
| 66 | return name in [ctn['RepoTags'][0] for ctn in cls.dckr.images()] |
| 67 | |
| 68 | @classmethod |
| 69 | def create_host_config(cls, port_list = None, host_guest_map = None, privileged = False): |
| 70 | port_bindings = None |
| 71 | binds = None |
| 72 | if port_list: |
| 73 | port_bindings = {} |
| 74 | for p in port_list: |
| 75 | port_bindings[str(p)] = str(p) |
| 76 | |
| 77 | if host_guest_map: |
| 78 | binds = [] |
| 79 | for h, g in host_guest_map: |
| 80 | binds.append('{0}:{1}'.format(h, g)) |
| 81 | |
| 82 | return cls.dckr.create_host_config(binds = binds, port_bindings = port_bindings, privileged = privileged) |
| 83 | |
| 84 | @classmethod |
| 85 | def cleanup(cls, image): |
| 86 | cnt_list = filter(lambda c: c['Image'] == image, cls.dckr.containers()) |
| 87 | for cnt in cnt_list: |
| 88 | print('Cleaning container %s' %cnt['Id']) |
| 89 | cls.dckr.kill(cnt['Id']) |
| 90 | cls.dckr.remove_container(cnt['Id'], force=True) |
| 91 | |
Chetan Gaonker | 7142a34 | 2016-04-07 14:53:12 -0700 | [diff] [blame^] | 92 | @classmethod |
| 93 | def remove_container(cls, name, force=True): |
| 94 | try: |
| 95 | cls.dckr.remove_container(name, force = force) |
| 96 | except: pass |
| 97 | |
Chetan Gaonker | 93e302d | 2016-04-05 10:51:07 -0700 | [diff] [blame] | 98 | def exists(self): |
| 99 | return '/{0}'.format(self.name) in list(flatten(n['Names'] for n in self.dckr.containers())) |
| 100 | |
| 101 | def img_exists(self): |
| 102 | return self.image_name in [ctn['RepoTags'][0] for ctn in self.dckr.images()] |
| 103 | |
| 104 | def ip(self): |
| 105 | cnt_list = filter(lambda c: c['Image'] == self.image_name, self.dckr.containers()) |
| 106 | cnt_settings = cnt_list.pop() |
| 107 | return cnt_settings['NetworkSettings']['Networks']['bridge']['IPAddress'] |
| 108 | |
| 109 | def kill(self, remove = True): |
| 110 | self.dckr.kill(self.name) |
| 111 | self.dckr.remove_container(self.name, force=True) |
| 112 | |
| 113 | def start(self, rm = True, ports = None, volumes = None, host_config = None, |
| 114 | environment = None, tty = False, stdin_open = True): |
| 115 | |
| 116 | if rm and self.exists(): |
| 117 | print('Removing container:', self.name) |
| 118 | self.dckr.remove_container(self.name, force=True) |
| 119 | |
| 120 | ctn = self.dckr.create_container(image=self.image_name, ports = ports, command=self.command, |
| 121 | detach=True, name=self.name, |
| 122 | environment = environment, |
| 123 | volumes = volumes, |
| 124 | host_config = host_config, stdin_open=stdin_open, tty = tty) |
| 125 | self.dckr.start(container=self.name) |
| 126 | if self.bridge: |
| 127 | self.connect_to_br() |
| 128 | self.id = ctn['Id'] |
| 129 | return ctn |
| 130 | |
| 131 | def connect_to_br(self): |
| 132 | with docker_netns(self.name) as pid: |
| 133 | ip = IPRoute() |
| 134 | br = ip.link_lookup(ifname=self.bridge) |
| 135 | if len(br) == 0: |
| 136 | ip.link_create(ifname=self.bridge, kind='bridge') |
| 137 | br = ip.link_lookup(ifname=self.bridge) |
| 138 | br = br[0] |
| 139 | ip.link('set', index=br, state='up') |
| 140 | |
| 141 | ifs = ip.link_lookup(ifname=self.name) |
| 142 | if len(ifs) > 0: |
| 143 | ip.link_remove(ifs[0]) |
| 144 | |
| 145 | ip.link_create(ifname=self.name, kind='veth', peer=pid) |
| 146 | host = ip.link_lookup(ifname=self.name)[0] |
| 147 | ip.link('set', index=host, master=br) |
| 148 | ip.link('set', index=host, state='up') |
| 149 | guest = ip.link_lookup(ifname=pid)[0] |
| 150 | ip.link('set', index=guest, net_ns_fd=pid) |
| 151 | with Namespace(pid, 'net'): |
| 152 | ip = IPRoute() |
| 153 | ip.link('set', index=guest, ifname='eth1') |
| 154 | ip.link('set', index=guest, state='up') |
| 155 | ip.addr('add', index=guest, address=self.ipaddress, mask=self.mask) |
| 156 | |
| 157 | def execute(self, cmd, tty = True, stream = False, shell = False): |
| 158 | res = 0 |
| 159 | if type(cmd) == str: |
| 160 | cmds = (cmd,) |
| 161 | else: |
| 162 | cmds = cmd |
| 163 | if shell: |
| 164 | for c in cmds: |
| 165 | res += os.system('docker exec {0} {1}'.format(self.name, c)) |
| 166 | return res |
| 167 | for c in cmds: |
| 168 | i = self.dckr.exec_create(container=self.name, cmd=c, tty = tty, privileged = True) |
| 169 | self.dckr.exec_start(i['Id'], stream = stream) |
| 170 | result = self.dckr.exec_inspect(i['Id']) |
| 171 | res += 0 if result['ExitCode'] == None else result['ExitCode'] |
| 172 | return res |
| 173 | |
| 174 | class Onos(Container): |
| 175 | |
| 176 | quagga_config = { 'bridge' : 'quagga-br', 'ip': '10.10.0.4', 'mask' : 16 } |
| 177 | env = { 'ONOS_APPS' : 'drivers,openflow,proxyarp,aaa,igmp,vrouter' } |
| 178 | ports = [ 8181, 8101, 9876, 6653, 6633, 2000, 2620 ] |
| 179 | |
| 180 | def __init__(self, name = 'cord-onos', image = 'onosproject/onos', tag = 'latest', boot_delay = 60): |
| 181 | super(Onos, self).__init__(name, image, tag = tag, quagga_config = self.quagga_config) |
| 182 | if not self.exists(): |
Chetan Gaonker | 7142a34 | 2016-04-07 14:53:12 -0700 | [diff] [blame^] | 183 | self.remove_container(name, force=True) |
Chetan Gaonker | 93e302d | 2016-04-05 10:51:07 -0700 | [diff] [blame] | 184 | host_config = self.create_host_config(port_list = self.ports) |
| 185 | print('Starting ONOS container %s' %self.name) |
| 186 | self.start(ports = self.ports, environment = self.env, |
| 187 | host_config = host_config, tty = True) |
| 188 | print('Waiting %d seconds for ONOS to boot' %(boot_delay)) |
| 189 | time.sleep(boot_delay) |
| 190 | |
| 191 | class Radius(Container): |
| 192 | ports = [ 1812, 1813 ] |
| 193 | env = {'TIMEZONE':'America/Los_Angeles', |
| 194 | 'DEBUG': 'true', 'cert_password':'whatever', 'primary_shared_secret':'radius_password' |
| 195 | } |
| 196 | host_db_dir = os.path.join(os.getenv('HOME'), 'services', 'radius', 'data', 'db') |
| 197 | guest_db_dir = os.path.join(os.path.sep, 'opt', 'db') |
| 198 | host_config_dir = os.path.join(os.getenv('HOME'), 'services', 'radius', 'freeradius') |
| 199 | guest_config_dir = os.path.join(os.path.sep, 'etc', 'freeradius') |
| 200 | start_command = '/root/start-radius.py' |
| 201 | host_guest_map = ( (host_db_dir, guest_db_dir), |
| 202 | (host_config_dir, guest_config_dir) |
| 203 | ) |
| 204 | def __init__(self, name = 'cord-radius', image = 'freeradius', tag = 'podd'): |
| 205 | super(Radius, self).__init__(name, image, tag = tag, command = self.start_command) |
| 206 | if not self.exists(): |
Chetan Gaonker | 7142a34 | 2016-04-07 14:53:12 -0700 | [diff] [blame^] | 207 | self.remove_container(name, force=True) |
Chetan Gaonker | 93e302d | 2016-04-05 10:51:07 -0700 | [diff] [blame] | 208 | host_config = self.create_host_config(port_list = self.ports, |
| 209 | host_guest_map = self.host_guest_map) |
| 210 | volumes = [] |
| 211 | for h,g in self.host_guest_map: |
| 212 | volumes.append(g) |
| 213 | self.start(ports = self.ports, environment = self.env, |
| 214 | volumes = volumes, |
| 215 | host_config = host_config, tty = True) |
| 216 | |
| 217 | class CordTester(Container): |
| 218 | |
| 219 | sandbox = '/root/test' |
Chetan Gaonker | 7142a34 | 2016-04-07 14:53:12 -0700 | [diff] [blame^] | 220 | sandbox_setup = '/root/test/src/test/setup' |
| 221 | tester_paths = os.path.realpath(sys.argv[0]).split(os.path.sep) |
| 222 | tester_path_index = tester_paths.index('cord-tester') |
| 223 | sandbox_host = os.path.sep.join(tester_paths[:tester_path_index+1]) |
Chetan Gaonker | 93e302d | 2016-04-05 10:51:07 -0700 | [diff] [blame] | 224 | |
| 225 | host_guest_map = ( (sandbox_host, sandbox), |
| 226 | ('/lib/modules', '/lib/modules') |
| 227 | ) |
| 228 | basename = 'cord-tester' |
| 229 | |
| 230 | def __init__(self, image = 'cord-test/nose', tag = 'latest', env = None, rm = False, boot_delay=2): |
Chetan Gaonker | 93e302d | 2016-04-05 10:51:07 -0700 | [diff] [blame] | 231 | self.rm = rm |
| 232 | self.name = self.get_name() |
| 233 | super(CordTester, self).__init__(self.name, image = image, tag = tag) |
| 234 | host_config = self.create_host_config(host_guest_map = self.host_guest_map, privileged = True) |
| 235 | volumes = [] |
| 236 | for h, g in self.host_guest_map: |
| 237 | volumes.append(g) |
Chetan Gaonker | 7142a34 | 2016-04-07 14:53:12 -0700 | [diff] [blame^] | 238 | ##Remove test container if any |
| 239 | self.remove_container(self.name, force=True) |
Chetan Gaonker | 93e302d | 2016-04-05 10:51:07 -0700 | [diff] [blame] | 240 | print('Starting test container %s, image %s, tag %s' %(self.name, self.image, self.tag)) |
| 241 | self.start(rm = False, volumes = volumes, environment = env, |
| 242 | host_config = host_config, tty = True) |
Chetan Gaonker | 7142a34 | 2016-04-07 14:53:12 -0700 | [diff] [blame^] | 243 | ovs_cmd = os.path.join(self.sandbox_setup, 'of-bridge.sh') + ' br0' |
Chetan Gaonker | 93e302d | 2016-04-05 10:51:07 -0700 | [diff] [blame] | 244 | print('Starting OVS on test container %s' %self.name) |
| 245 | self.execute(ovs_cmd) |
| 246 | status = 1 |
| 247 | ## Wait for the LLDP flows to be added to the switch |
| 248 | tries = 0 |
| 249 | while status != 0 and tries < 100: |
| 250 | cmd = 'ovs-ofctl dump-flows br0 | grep \"type=0x8942\"' |
| 251 | status = self.execute(cmd, shell = True) |
| 252 | tries += 1 |
| 253 | if tries % 10 == 0: |
| 254 | print('Waiting for test switch to be connected to ONOS controller ...') |
| 255 | |
| 256 | if status != 0: |
| 257 | print('Test Switch not connected to ONOS container.' |
| 258 | 'Please remove ONOS container and restart the test') |
| 259 | if self.rm: |
| 260 | self.kill() |
| 261 | sys.exit(1) |
| 262 | |
| 263 | time.sleep(boot_delay) |
| 264 | |
| 265 | @classmethod |
| 266 | def get_name(cls): |
| 267 | cnt_name = '/{0}'.format(cls.basename) |
| 268 | cnt_name_len = len(cnt_name) |
| 269 | names = list(flatten(n['Names'] for n in cls.dckr.containers(all=True))) |
| 270 | test_names = filter(lambda n: n.startswith(cnt_name), names) |
| 271 | last_cnt_number = 0 |
| 272 | if test_names: |
| 273 | last_cnt_name = reduce(lambda n1, n2: n1 if int(n1[cnt_name_len:]) > \ |
| 274 | int(n2[cnt_name_len:]) else n2, |
| 275 | test_names) |
| 276 | last_cnt_number = int(last_cnt_name[cnt_name_len:]) |
| 277 | test_cnt_name = cls.basename + str(last_cnt_number+1) |
| 278 | return test_cnt_name |
| 279 | |
| 280 | @classmethod |
| 281 | def build_image(cls, image): |
| 282 | print('Building test container docker image %s' %image) |
| 283 | dockerfile = ''' |
| 284 | FROM ubuntu:14.04 |
| 285 | MAINTAINER chetan@ciena.com |
| 286 | RUN apt-get update |
| 287 | RUN apt-get -y install git python python-pip python-setuptools python-scapy tcpdump doxygen doxypy wget |
| 288 | RUN easy_install nose |
| 289 | RUN apt-get -y install openvswitch-common openvswitch-switch |
| 290 | RUN mkdir -p /root/ovs |
| 291 | WORKDIR /root |
| 292 | RUN wget http://openvswitch.org/releases/openvswitch-2.4.0.tar.gz -O /root/ovs/openvswitch-2.4.0.tar.gz && \ |
| 293 | (cd /root/ovs && tar zxpvf openvswitch-2.4.0.tar.gz && \ |
| 294 | cd openvswitch-2.4.0 && \ |
| 295 | ./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --disable-ssl && make && make install) |
| 296 | RUN service openvswitch-switch restart || /bin/true |
| 297 | RUN apt-get -y install python-twisted python-sqlite sqlite3 |
| 298 | RUN pip install scapy-ssl_tls |
| 299 | RUN pip install -U scapy |
| 300 | RUN pip install monotonic |
| 301 | RUN mv /usr/sbin/tcpdump /sbin/ |
| 302 | RUN ln -sf /sbin/tcpdump /usr/sbin/tcpdump |
| 303 | CMD ["/bin/bash"] |
| 304 | ''' |
| 305 | super(CordTester, cls).build_image(dockerfile, image) |
| 306 | print('Done building docker image %s' %image) |
| 307 | |
| 308 | def run_tests(self, tests): |
| 309 | '''Run the list of tests''' |
| 310 | for t in tests: |
| 311 | test = t.split(':')[0] |
| 312 | if test == 'tls': |
| 313 | test_file = test + 'AuthTest.py' |
| 314 | else: |
| 315 | test_file = test + 'Test.py' |
| 316 | |
| 317 | if t.find(':') >= 0: |
| 318 | test_case = test_file + ':' + t.split(':')[1] |
| 319 | else: |
| 320 | test_case = test_file |
Chetan Gaonker | 7142a34 | 2016-04-07 14:53:12 -0700 | [diff] [blame^] | 321 | cmd = 'nosetests -v {0}/src/test/{1}/{2}'.format(self.sandbox, test, test_case) |
Chetan Gaonker | 93e302d | 2016-04-05 10:51:07 -0700 | [diff] [blame] | 322 | status = self.execute(cmd, shell = True) |
| 323 | print('Test %s %s' %(test_case, 'Success' if status == 0 else 'Failure')) |
| 324 | print('Done running tests') |
| 325 | if self.rm: |
| 326 | print('Removing test container %s' %self.name) |
| 327 | self.kill(remove=True) |
| 328 | |
| 329 | |
| 330 | ##default onos/radius/test container images and names |
| 331 | onos_image_default='onosproject/onos:latest' |
| 332 | nose_image_default='cord-test/nose:latest' |
| 333 | test_type_default='dhcp' |
| 334 | onos_app_version = '1.0-SNAPSHOT' |
Chetan Gaonker | 7142a34 | 2016-04-07 14:53:12 -0700 | [diff] [blame^] | 335 | onos_tester_base = os.path.dirname(os.path.realpath(sys.argv[0])) |
| 336 | onos_app_file = os.path.abspath('{0}/../apps/ciena-cordigmp-'.format(onos_tester_base) + onos_app_version + '.oar') |
Chetan Gaonker | 93e302d | 2016-04-05 10:51:07 -0700 | [diff] [blame] | 337 | zebra_quagga_config = { 'bridge' : 'quagga-br', 'ip': '10.10.0.1', 'mask': 16 } |
| 338 | |
| 339 | def runTest(args): |
| 340 | onos_cnt = {'tag':'latest'} |
| 341 | radius_cnt = {'tag':'latest'} |
| 342 | nose_cnt = {'image': 'cord-test/nose','tag': 'latest'} |
| 343 | |
| 344 | #print('Test type %s, onos %s, radius %s, app %s, olt %s, cleanup %s, kill flag %s, build image %s' |
| 345 | # %(args.test_type, args.onos, args.radius, args.app, args.olt, args.cleanup, args.kill, args.build)) |
| 346 | if args.cleanup: |
| 347 | cleanup_container = args.cleanup |
| 348 | if cleanup_container.find(':') < 0: |
| 349 | cleanup_container += ':latest' |
| 350 | print('Cleaning up containers %s' %cleanup_container) |
| 351 | Container.cleanup(cleanup_container) |
| 352 | sys.exit(0) |
| 353 | |
| 354 | onos_cnt['image'] = args.onos.split(':')[0] |
| 355 | if args.onos.find(':') >= 0: |
| 356 | onos_cnt['tag'] = args.onos.split(':')[1] |
| 357 | |
| 358 | onos = Onos(image = onos_cnt['image'], tag = onos_cnt['tag'], boot_delay = 60) |
| 359 | onos_ip = onos.ip() |
| 360 | |
| 361 | ##Start Radius container if specified |
| 362 | if args.radius: |
| 363 | radius_cnt['image'] = args.radius.split(':')[0] |
| 364 | if args.radius.find(':') >= 0: |
| 365 | radius_cnt['tag'] = args.radius.split(':')[1] |
| 366 | radius = Radius(image = radius_cnt['image'], tag = radius_cnt['tag']) |
| 367 | radius_ip = radius.ip() |
| 368 | print('Started Radius server with IP %s' %radius_ip) |
| 369 | else: |
| 370 | radius_ip = None |
| 371 | |
| 372 | print('Onos IP %s, Test type %s' %(onos_ip, args.test_type)) |
| 373 | print('Installing ONOS app %s' %onos_app_file) |
| 374 | |
| 375 | OnosCtrl.install_app(args.app) |
| 376 | |
| 377 | build_cnt_image = args.build.strip() |
| 378 | if build_cnt_image: |
| 379 | CordTester.build_image(build_cnt_image) |
| 380 | nose_cnt['image']= build_cnt_image.split(':')[0] |
| 381 | if build_cnt_image.find(':') >= 0: |
| 382 | nose_cnt['tag'] = build_cnt_image.split(':')[1] |
| 383 | |
| 384 | test_cnt_env = { 'ONOS_CONTROLLER_IP' : onos_ip, |
| 385 | 'ONOS_AAA_IP' : radius_ip, |
| 386 | } |
| 387 | if args.olt: |
Chetan Gaonker | 7142a34 | 2016-04-07 14:53:12 -0700 | [diff] [blame^] | 388 | olt_conf_test_loc = os.path.join(CordTester.sandbox_setup, 'olt_config.json') |
Chetan Gaonker | 93e302d | 2016-04-05 10:51:07 -0700 | [diff] [blame] | 389 | test_cnt_env['OLT_CONFIG'] = olt_conf_test_loc |
| 390 | |
| 391 | test_cnt = CordTester(image = nose_cnt['image'], tag = nose_cnt['tag'], |
| 392 | env = test_cnt_env, |
| 393 | rm = args.kill) |
| 394 | tests = args.test_type.split('-') |
| 395 | test_cnt.run_tests(tests) |
| 396 | |
| 397 | if __name__ == '__main__': |
| 398 | parser = ArgumentParser(description='Cord Tester for ONOS') |
| 399 | parser.add_argument('-t', '--test-type', default=test_type_default, type=str) |
| 400 | parser.add_argument('-o', '--onos', default=onos_image_default, type=str, help='ONOS container image') |
| 401 | parser.add_argument('-r', '--radius',default='',type=str, help='Radius container image') |
| 402 | parser.add_argument('-a', '--app', default=onos_app_file, type=str, help='Cord ONOS app filename') |
| 403 | parser.add_argument('-l', '--olt', action='store_true', help='Use OLT config') |
| 404 | parser.add_argument('-c', '--cleanup', default='', type=str, help='Cleanup test containers') |
| 405 | parser.add_argument('-k', '--kill', action='store_true', help='Remove test container after tests') |
| 406 | parser.add_argument('-b', '--build', default='', type=str) |
| 407 | parser.set_defaults(func=runTest) |
| 408 | args = parser.parse_args() |
| 409 | args.func(args) |