blob: b96c009370762d5dc3038a54a7d3de6bfcc641e0 [file] [log] [blame]
Dan Talayco48370102010-03-03 15:17:33 -08001#!/usr/bin/env python
2"""
3@package oft
4
5OpenFlow test framework top level script
6
Rich Laneea5060d2013-01-06 13:59:00 -08007This script is the entry point for running OpenFlow tests using the OFT
8framework. For usage information, see --help or the README.
Dan Talayco48370102010-03-03 15:17:33 -08009
Stephen Finucane7424a742014-05-15 19:46:06 +010010To add a new command line option, edit both the CONFIG_DEFAULT dictionary and
Rich Laneea5060d2013-01-06 13:59:00 -080011the config_setup function. The option's result will end up in the global
12oftest.config dictionary.
Dan Talayco48370102010-03-03 15:17:33 -080013"""
14
Stephen Finucane01588652014-05-15 20:03:12 +010015from __future__ import print_function
16
Dan Talayco48370102010-03-03 15:17:33 -080017import sys
Rich Lane95f078b2013-01-06 13:24:58 -080018import optparse
Dan Talayco48370102010-03-03 15:17:33 -080019import logging
20import unittest
Dan Talayco2c0dba32010-03-06 22:47:06 -080021import time
Brandon Heller446c1432010-04-01 12:43:27 -070022import os
Rich Lane6b452bc2012-07-09 16:52:21 -070023import imp
Rich Lane8592bec2012-09-03 09:06:59 -070024import random
Rich Lane5bd6cf92012-10-04 17:57:24 -070025import signal
Rich Lane943be672012-10-04 19:20:16 -070026import fnmatch
Rich Lane2d6d4822013-01-08 10:49:16 -080027import copy
Dan Talayco48370102010-03-03 15:17:33 -080028
Stephen Finucane7424a742014-05-15 19:46:06 +010029ROOT_DIR = os.path.dirname(os.path.realpath(__file__))
Rich Lanefadf3452012-10-03 16:23:37 -070030
Stephen Finucane7424a742014-05-15 19:46:06 +010031PY_SRC_DIR = os.path.join(ROOT_DIR, 'src', 'python')
32if os.path.exists(os.path.join(PY_SRC_DIR, 'oftest')):
Rich Lane39878042012-07-09 14:45:35 -070033 # Running from source tree
Stephen Finucane7424a742014-05-15 19:46:06 +010034 sys.path.insert(0, PY_SRC_DIR)
Rich Lane39878042012-07-09 14:45:35 -070035
Rich Lane477f4812012-10-04 22:49:00 -070036import oftest
37from oftest import config
Rich Lanee55abf72012-07-26 20:11:42 -070038import oftest.ofutils
Rich Lane95f078b2013-01-06 13:24:58 -080039import oftest.help_formatter
Rich Lane3f7098c2013-03-12 10:28:32 -070040import loxi
Dan Talaycoba3745c2010-07-21 21:51:08 -070041
Dan Talayco48370102010-03-03 15:17:33 -080042##@var DEBUG_LEVELS
43# Map from strings to debugging levels
44DEBUG_LEVELS = {
45 'debug' : logging.DEBUG,
46 'verbose' : logging.DEBUG,
47 'info' : logging.INFO,
48 'warning' : logging.WARNING,
49 'warn' : logging.WARNING,
50 'error' : logging.ERROR,
51 'critical' : logging.CRITICAL
52}
53
Stephen Finucane7424a742014-05-15 19:46:06 +010054##@var CONFIG_DEFAULT
Dan Talayco48370102010-03-03 15:17:33 -080055# The default configuration dictionary for OFT
Stephen Finucane7424a742014-05-15 19:46:06 +010056CONFIG_DEFAULT = {
Rich Lane95f078b2013-01-06 13:24:58 -080057 # Miscellaneous options
58 "list" : False,
59 "list_test_names" : False,
60 "allow_user" : False,
61
62 # Test selection options
Rich Lanec76b09a2013-01-02 16:53:22 -080063 "test_spec" : "",
Rich Lane1a08d232013-01-04 16:03:50 -080064 "test_file" : None,
Rich Lane74b13d12013-05-03 17:58:50 -070065 "test_dir" : None,
Rich Lane95f078b2013-01-06 13:24:58 -080066
67 # Switch connection options
68 "controller_host" : "0.0.0.0", # For passive bind
Rich Lane4d1f3eb2013-10-03 13:45:57 -070069 "controller_port" : 6653,
Rich Lane95f078b2013-01-06 13:24:58 -080070 "switch_ip" : None, # If not none, actively connect to switch
Rich Lane15f26322013-01-08 11:23:24 -080071 "platform" : "eth",
Rich Lane95f078b2013-01-06 13:24:58 -080072 "platform_args" : None,
Stephen Finucane7424a742014-05-15 19:46:06 +010073 "platform_dir" : os.path.join(ROOT_DIR, "platforms"),
Rich Lane2d6d4822013-01-08 10:49:16 -080074 "interfaces" : [],
Rich Lane9fd05682013-01-10 15:30:38 -080075 "openflow_version" : "1.0",
Rich Lane95f078b2013-01-06 13:24:58 -080076
77 # Logging options
Dan Talayco48370102010-03-03 15:17:33 -080078 "log_file" : "oft.log",
Rich Lane69fd8e02013-08-23 16:23:42 -070079 "log_dir" : None,
Rich Lane95f078b2013-01-06 13:24:58 -080080 "debug" : "verbose",
Rich Lane9631f002014-03-21 18:05:16 -070081 "profile" : False,
Stephen Finucane6219af72014-05-14 21:08:30 +010082 "profile_file" : "profile.out",
Stephen Finucanee016cf22014-04-16 22:04:11 +010083 "xunit" : False,
84 "xunit_dir" : "xunit",
Rich Lane95f078b2013-01-06 13:24:58 -080085
86 # Test behavior options
87 "relax" : False,
Dan Talaycod7c80d12012-04-03 15:20:57 -070088 "test_params" : "None",
Rich Lane9a84a4f2012-07-17 12:27:42 -070089 "fail_skipped" : False,
Rich Lane48f6aed2014-03-23 15:51:02 -070090 "default_timeout" : 2.0,
91 "default_negative_timeout" : 0.01,
Jeffrey Townsend5cb7ed32012-08-17 18:11:01 +000092 "minsize" : 0,
Rich Lane8592bec2012-09-03 09:06:59 -070093 "random_seed" : None,
Stephen Finucane92f7cf62014-03-13 15:08:11 +000094 "disable_ipv6" : False,
Rich Lane62b4fb12015-04-02 18:04:10 -070095 "random_order" : False,
Rich Lane95f078b2013-01-06 13:24:58 -080096
97 # Other configuration
Rich Lane15f26322013-01-08 11:23:24 -080098 "port_map" : {},
Dan Talayco48370102010-03-03 15:17:33 -080099}
100
Rich Lane95f078b2013-01-06 13:24:58 -0800101def config_setup():
Dan Talayco48370102010-03-03 15:17:33 -0800102 """
103 Set up the configuration including parsing the arguments
104
Dan Talayco48370102010-03-03 15:17:33 -0800105 @return A pair (config, args) where config is an config
106 object and args is any additional arguments from the command line
107 """
108
Rich Lane4113a582013-01-03 10:13:02 -0800109 usage = "usage: %prog [options] (test|group)..."
110
Rich Lane95f078b2013-01-06 13:24:58 -0800111 description = """\
Rich Lane4113a582013-01-03 10:13:02 -0800112OFTest is a framework and set of tests for validating OpenFlow switches.
113
114The default configuration assumes that an OpenFlow 1.0 switch is attempting to
Rich Lane4d1f3eb2013-10-03 13:45:57 -0700115connect to a controller on the machine running OFTest, port 6653. Additionally,
Rich Lane4113a582013-01-03 10:13:02 -0800116the interfaces veth1, veth3, veth5, and veth7 should be connected to the switch's
117dataplane.
118
119If no positional arguments are given then OFTest will run all tests that
120depend only on standard OpenFlow 1.0. Otherwise each positional argument
121is interpreted as either a test name or a test group name. The union of
122these will be executed. To see what groups each test belongs to use the
Rich Lane948f0302013-01-07 10:59:08 -0800123--list option. Tests and groups can be subtracted from the result by
124prefixing them with the '^' character.
Rich Lane4113a582013-01-03 10:13:02 -0800125"""
126
Rich Lane2d6d4822013-01-08 10:49:16 -0800127 # Parse --interface
128 def check_interface(option, opt, value):
129 try:
130 ofport, interface = value.split('@', 1)
131 ofport = int(ofport)
132 except ValueError:
133 raise optparse.OptionValueError("incorrect interface syntax (got %s, expected 'ofport@interface')" % repr(value))
134 return (ofport, interface)
135
136 class Option(optparse.Option):
137 TYPES = optparse.Option.TYPES + ("interface",)
138 TYPE_CHECKER = copy.copy(optparse.Option.TYPE_CHECKER)
139 TYPE_CHECKER["interface"] = check_interface
140
Rich Lane95f078b2013-01-06 13:24:58 -0800141 parser = optparse.OptionParser(version="%prog 0.1",
142 usage=usage,
143 description=description,
Rich Lane2d6d4822013-01-08 10:49:16 -0800144 formatter=oftest.help_formatter.HelpFormatter(),
145 option_class=Option)
Dan Talayco48370102010-03-03 15:17:33 -0800146
147 # Set up default values
Stephen Finucane7424a742014-05-15 19:46:06 +0100148 parser.set_defaults(**CONFIG_DEFAULT)
Dan Talayco48370102010-03-03 15:17:33 -0800149
Dan Talayco2c0dba32010-03-06 22:47:06 -0800150 parser.add_option("--list", action="store_true",
Brandon Heller824504e2010-04-01 12:21:37 -0700151 help="List all tests and exit")
Rich Lane95f078b2013-01-06 13:24:58 -0800152 parser.add_option("--list-test-names", action='store_true',
153 help="List test names matching the test spec and exit")
Rich Laneee57ad02012-07-13 15:40:36 -0700154 parser.add_option("--allow-user", action="store_true",
155 help="Proceed even if oftest is not run as root")
Rich Lane95f078b2013-01-06 13:24:58 -0800156
157 group = optparse.OptionGroup(parser, "Test selection options")
158 group.add_option("-T", "--test-spec", "--test-list", help="Tests to run, separated by commas")
159 group.add_option("-f", "--test-file", help="File of tests to run, one per line")
160 group.add_option("--test-dir", type="string", help="Directory containing tests")
161 parser.add_option_group(group)
162
163 group = optparse.OptionGroup(parser, "Switch connection options")
164 group.add_option("-H", "--host", dest="controller_host",
165 help="IP address to listen on (default %default)")
166 group.add_option("-p", "--port", dest="controller_port",
167 type="int", help="Port number to listen on (default %default)")
168 group.add_option("-S", "--switch-ip", dest="switch_ip",
169 help="If set, actively connect to this switch by IP")
170 group.add_option("-P", "--platform", help="Platform module name (default %default)")
171 group.add_option("-a", "--platform-args", help="Custom arguments for the platform")
172 group.add_option("--platform-dir", type="string", help="Directory containing platform modules")
Rich Lane2d6d4822013-01-08 10:49:16 -0800173 group.add_option("--interface", "-i", type="interface", dest="interfaces", metavar="INTERFACE", action="append",
174 help="Specify a OpenFlow port number and the dataplane interface to use. May be given multiple times. Example: 1@eth1")
Rich Lane50cfa502013-04-25 13:45:08 -0700175 group.add_option("--of-version", "-V", dest="openflow_version", choices=loxi.version_names.values(),
Rich Lane9fd05682013-01-10 15:30:38 -0800176 help="OpenFlow version to use")
Rich Lane95f078b2013-01-06 13:24:58 -0800177 parser.add_option_group(group)
178
179 group = optparse.OptionGroup(parser, "Logging options")
Rich Lane69fd8e02013-08-23 16:23:42 -0700180 group.add_option("--log-file", help="Name of log file (default %default)")
181 group.add_option("--log-dir", help="Name of log directory")
Rich Lane95f078b2013-01-06 13:24:58 -0800182 dbg_lvl_names = sorted(DEBUG_LEVELS.keys(), key=lambda x: DEBUG_LEVELS[x])
183 group.add_option("--debug", choices=dbg_lvl_names,
184 help="Debug lvl: debug, info, warning, error, critical (default %default)")
185 group.add_option("-v", "--verbose", action="store_const", dest="debug",
186 const="verbose", help="Shortcut for --debug=verbose")
187 group.add_option("-q", "--quiet", action="store_const", dest="debug",
188 const="warning", help="Shortcut for --debug=warning")
Stephen Finucane6219af72014-05-14 21:08:30 +0100189 group.add_option("--profile", action="store_true", help="Enable Python profiling")
190 group.add_option("--profile-file", help="Output file for Python profiler")
Stephen Finucanee016cf22014-04-16 22:04:11 +0100191 group.add_option("--xunit", action="store_true", help="Enable xUnit-formatted results")
192 group.add_option("--xunit-dir", help="Output directory for xUnit-formatted results")
Rich Lane95f078b2013-01-06 13:24:58 -0800193 parser.add_option_group(group)
194
195 group = optparse.OptionGroup(parser, "Test behavior options")
196 group.add_option("--relax", action="store_true",
197 help="Relax packet match checks allowing other packets")
198 test_params_help = """Set test parameters: key=val;... (see --list)
199 """
200 group.add_option("-t", "--test-params", help=test_params_help)
201 group.add_option("--fail-skipped", action="store_true",
Rich Lane9a84a4f2012-07-17 12:27:42 -0700202 help="Return failure if any test was skipped")
Rich Lane48f6aed2014-03-23 15:51:02 -0700203 group.add_option("--default-timeout", type=float,
Rich Lanee55abf72012-07-26 20:11:42 -0700204 help="Timeout in seconds for most operations")
Rich Lane48f6aed2014-03-23 15:51:02 -0700205 group.add_option("--default-negative-timeout", type=float,
206 help="Timeout in seconds for negative checks")
Rich Lane95f078b2013-01-06 13:24:58 -0800207 group.add_option("--minsize", type="int",
208 help="Minimum allowable packet size on the dataplane.")
209 group.add_option("--random-seed", type="int",
210 help="Random number generator seed")
Stephen Finucane92f7cf62014-03-13 15:08:11 +0000211 group.add_option("--disable-ipv6", action="store_true",
212 help="Disable IPv6 tests")
Rich Lane62b4fb12015-04-02 18:04:10 -0700213 group.add_option("--random-order", action="store_true",
214 help="Randomize order of tests")
Rich Lane95f078b2013-01-06 13:24:58 -0800215 parser.add_option_group(group)
Jeffrey Townsend5cb7ed32012-08-17 18:11:01 +0000216
Dan Talayco48370102010-03-03 15:17:33 -0800217 # Might need this if other parsers want command line
218 # parser.allow_interspersed_args = False
219 (options, args) = parser.parse_args()
220
Rich Lane74b13d12013-05-03 17:58:50 -0700221 # If --test-dir wasn't given, pick one based on the OpenFlow version
222 if options.test_dir == None:
223 if options.openflow_version == "1.0":
Stephen Finucane7424a742014-05-15 19:46:06 +0100224 options.test_dir = os.path.join(ROOT_DIR, "tests")
Rich Lane74b13d12013-05-03 17:58:50 -0700225 else:
Stephen Finucane7424a742014-05-15 19:46:06 +0100226 options.test_dir = os.path.join(ROOT_DIR, "tests-" + options.openflow_version)
Rich Lane74b13d12013-05-03 17:58:50 -0700227
Rich Lane95f078b2013-01-06 13:24:58 -0800228 # Convert options from a Namespace to a plain dictionary
Stephen Finucane7424a742014-05-15 19:46:06 +0100229 config = CONFIG_DEFAULT.copy()
Rich Lane95f078b2013-01-06 13:24:58 -0800230 for key in config.keys():
231 config[key] = getattr(options, key)
Dan Talayco48370102010-03-03 15:17:33 -0800232
233 return (config, args)
234
235def logging_setup(config):
236 """
237 Set up logging based on config
238 """
Rich Lane69fd8e02013-08-23 16:23:42 -0700239
240 logging.getLogger().setLevel(DEBUG_LEVELS[config["debug"]])
241
242 if config["log_dir"] != None:
243 if os.path.exists(config["log_dir"]):
244 import shutil
245 shutil.rmtree(config["log_dir"])
246 os.makedirs(config["log_dir"])
247 else:
248 if os.path.exists(config["log_file"]):
249 os.remove(config["log_file"])
250
251 oftest.open_logfile('main')
Dan Talayco48370102010-03-03 15:17:33 -0800252
Stephen Finucanee016cf22014-04-16 22:04:11 +0100253def xunit_setup(config):
254 """
255 Set up xUnit output based on config
256 """
257
258 if not config["xunit"]:
259 return
260
Stephen Finucane6219af72014-05-14 21:08:30 +0100261 if os.path.exists(config["xunit_dir"]):
262 import shutil
263 shutil.rmtree(config["xunit_dir"])
264 os.makedirs(config["xunit_dir"])
Stephen Finucanee016cf22014-04-16 22:04:11 +0100265
Rich Lane472aaea2013-08-27 09:27:38 -0700266def pcap_setup(config):
267 """
268 Set up dataplane packet capturing based on config
269 """
270
271 if config["log_dir"] == None:
272 filename = os.path.splitext(config["log_file"])[0] + '.pcap'
273 oftest.dataplane_instance.start_pcap(filename)
274 else:
275 # start_pcap is called per-test in base_tests
276 pass
277
Stephen Finucane6219af72014-05-14 21:08:30 +0100278def profiler_setup(config):
279 """
280 Set up profiler based on config
281 """
282
283 if not config["profile"]:
284 return
285
286 import cProfile
287 profiler = cProfile.Profile()
288 profiler.enable()
289
290 return profiler
291
292def profiler_teardown(profiler):
293 """
294 Tear down profiler based on config
295 """
296
297 if not config["profile"]:
298 return
299
300 profiler.disable()
301 profiler.dump_stats(config["profile_file"])
302
Rich Lane472aaea2013-08-27 09:27:38 -0700303
Rich Lane943be672012-10-04 19:20:16 -0700304def load_test_modules(config):
305 """
306 Load tests from the test_dir directory.
Dan Talayco2c0dba32010-03-06 22:47:06 -0800307
Rich Lane943be672012-10-04 19:20:16 -0700308 Test cases are subclasses of unittest.TestCase
Dan Talayco2c0dba32010-03-06 22:47:06 -0800309
Rich Lanecc45b8e2013-01-02 15:55:02 -0800310 Also updates the _groups member to include "standard" and
311 module test groups if appropriate.
312
Dan Talayco2c0dba32010-03-06 22:47:06 -0800313 @param config The oft configuration dictionary
Rich Lane943be672012-10-04 19:20:16 -0700314 @returns A dictionary from test module names to tuples of
315 (module, dictionary from test names to test classes).
Dan Talayco2c0dba32010-03-06 22:47:06 -0800316 """
317
Rich Lane943be672012-10-04 19:20:16 -0700318 result = {}
Dan Talayco2c0dba32010-03-06 22:47:06 -0800319
Rich Lane943be672012-10-04 19:20:16 -0700320 for root, dirs, filenames in os.walk(config["test_dir"]):
321 # Iterate over each python file
322 for filename in fnmatch.filter(filenames, '[!.]*.py'):
323 modname = os.path.splitext(os.path.basename(filename))[0]
Dan Talayco2c0dba32010-03-06 22:47:06 -0800324
Rich Lane943be672012-10-04 19:20:16 -0700325 try:
326 if sys.modules.has_key(modname):
327 mod = sys.modules[modname]
328 else:
329 mod = imp.load_module(modname, *imp.find_module(modname, [root]))
330 except:
331 logging.warning("Could not import file " + filename)
332 raise
Rich Lane520e4152012-07-09 16:18:16 -0700333
Rich Lane943be672012-10-04 19:20:16 -0700334 # Find all testcases defined in the module
335 tests = dict((k, v) for (k, v) in mod.__dict__.items() if type(v) == type and
Rich Lane9cef2742013-07-16 13:27:00 -0700336 issubclass(v, unittest.TestCase) and
337 hasattr(v, "runTest"))
Rich Lane943be672012-10-04 19:20:16 -0700338 if tests:
Rich Lanecc45b8e2013-01-02 15:55:02 -0800339 for (testname, test) in tests.items():
340 # Set default annotation values
341 if not hasattr(test, "_groups"):
342 test._groups = []
343 if not hasattr(test, "_nonstandard"):
344 test._nonstandard = False
345 if not hasattr(test, "_disabled"):
346 test._disabled = False
347
348 # Put test in its module's test group
349 if not test._disabled:
350 test._groups.append(modname)
351
352 # Put test in the standard test group
353 if not test._disabled and not test._nonstandard:
354 test._groups.append("standard")
355 test._groups.append("all") # backwards compatibility
356
Rich Lane943be672012-10-04 19:20:16 -0700357 result[modname] = (mod, tests)
Rich Lane520e4152012-07-09 16:18:16 -0700358
Rich Lane943be672012-10-04 19:20:16 -0700359 return result
Dan Talayco2c0dba32010-03-06 22:47:06 -0800360
Rich Lane5a9a1922013-01-11 14:29:30 -0800361def prune_tests(test_specs, test_modules, version):
Rich Lane15f64de2012-10-04 21:25:57 -0700362 """
Rich Lane5a9a1922013-01-11 14:29:30 -0800363 Return tests matching the given test-specs and OpenFlow version
Rich Lanec76b09a2013-01-02 16:53:22 -0800364 @param test_specs A list of group names or test names.
Rich Lane5a9a1922013-01-11 14:29:30 -0800365 @param version An OpenFlow version (e.g. "1.0")
Rich Lane15f64de2012-10-04 21:25:57 -0700366 @param test_modules Same format as the output of load_test_modules.
367 @returns Same format as the output of load_test_modules.
368 """
369 result = {}
Rich Lanec76b09a2013-01-02 16:53:22 -0800370 for e in test_specs:
Rich Lane15f64de2012-10-04 21:25:57 -0700371 matched = False
Rich Lane948f0302013-01-07 10:59:08 -0800372
373 if e.startswith('^'):
374 negated = True
375 e = e[1:]
376 else:
377 negated = False
378
Rich Lane15f64de2012-10-04 21:25:57 -0700379 for (modname, (mod, tests)) in test_modules.items():
Rich Lanecc45b8e2013-01-02 15:55:02 -0800380 for (testname, test) in tests.items():
381 if e in test._groups or e == "%s.%s" % (modname, testname):
382 result.setdefault(modname, (mod, {}))
Rich Lane948f0302013-01-07 10:59:08 -0800383 if not negated:
Rich Lane5a9a1922013-01-11 14:29:30 -0800384 if not hasattr(test, "_versions") or version in test._versions:
385 result[modname][1][testname] = test
Rich Lane948f0302013-01-07 10:59:08 -0800386 else:
387 if modname in result and testname in result[modname][1]:
388 del result[modname][1][testname]
389 if not result[modname][1]:
390 del result[modname]
Rich Lanecc45b8e2013-01-02 15:55:02 -0800391 matched = True
Rich Lane948f0302013-01-07 10:59:08 -0800392
Rich Lane15f64de2012-10-04 21:25:57 -0700393 if not matched:
Rich Lanecc45b8e2013-01-02 15:55:02 -0800394 die("test-spec element %s did not match any tests" % e)
Rich Lane948f0302013-01-07 10:59:08 -0800395
Rich Lane15f64de2012-10-04 21:25:57 -0700396 return result
397
Dan Talayco2c0dba32010-03-06 22:47:06 -0800398def die(msg, exit_val=1):
Dan Talayco2c0dba32010-03-06 22:47:06 -0800399 logging.critical(msg)
400 sys.exit(exit_val)
401
Dan Talayco48370102010-03-03 15:17:33 -0800402#
403# Main script
404#
405
Rich Lane477f4812012-10-04 22:49:00 -0700406# Setup global configuration
Rich Lane95f078b2013-01-06 13:24:58 -0800407(new_config, args) = config_setup()
Rich Lane477f4812012-10-04 22:49:00 -0700408oftest.config.update(new_config)
Dan Talayco48370102010-03-03 15:17:33 -0800409
Rich Laned7a85c42012-09-28 15:38:45 -0700410logging_setup(config)
Stephen Finucanee016cf22014-04-16 22:04:11 +0100411xunit_setup(config)
Rich Laned7a85c42012-09-28 15:38:45 -0700412logging.info("++++++++ " + time.asctime() + " ++++++++")
413
Rich Lane9fd05682013-01-10 15:30:38 -0800414# Pick an OpenFlow protocol module based on the configured version
Rich Lane50cfa502013-04-25 13:45:08 -0700415name_to_version = dict((v,k) for k, v in loxi.version_names.iteritems())
416sys.modules["ofp"] = loxi.protocol(name_to_version[config["openflow_version"]])
Rich Lane9fd05682013-01-10 15:30:38 -0800417
418# HACK: testutils.py imports controller.py, which needs the ofp module
419import oftest.testutils
420
Rich Lanee284b6b2012-10-03 09:19:58 -0700421# Allow tests to import each other
422sys.path.append(config["test_dir"])
423
Rich Lanec76b09a2013-01-02 16:53:22 -0800424test_specs = args
425if config["test_spec"] != "":
Stephen Finucane01588652014-05-15 20:03:12 +0100426 logging.warning("The '--test-spec' option is deprecated.")
Rich Lanec76b09a2013-01-02 16:53:22 -0800427 test_specs += config["test_spec"].split(',')
Rich Lane1a08d232013-01-04 16:03:50 -0800428if config["test_file"] != None:
429 with open(config["test_file"], 'r') as f:
430 for line in f:
431 line, _, _ = line.partition('#') # remove comments
432 line = line.strip()
433 if line:
434 test_specs.append(line)
Rich Lanec76b09a2013-01-02 16:53:22 -0800435if test_specs == []:
436 test_specs = ["standard"]
437
Rich Laned8e45482013-01-02 17:36:21 -0800438test_modules = load_test_modules(config)
Dan Talayco2c0dba32010-03-06 22:47:06 -0800439
440# Check if test list is requested; display and exit if so
441if config["list"]:
Dan Talayco7f8dba82012-04-12 12:58:52 -0700442 mod_count = 0
443 test_count = 0
Rich Lane37f42112013-01-03 13:41:49 -0800444 all_groups = set()
Stephen Finucane01588652014-05-15 20:03:12 +0100445
446 print("""
447Tests are shown grouped by module. If a test is in any groups beyond
448"standard" and its module's group then they are shown in parentheses.
449
Rich Lanee811e7b2013-01-03 13:36:54 -0800450Tests marked with '*' are non-standard and may require vendor extensions or
Stephen Finucane01588652014-05-15 20:03:12 +0100451special switch configuration. These are not part of the "standard" test group.
452
453Tests marked with '!' are disabled because they are experimental,
454special-purpose, or are too long to be run normally. These are not part of
455the "standard" test group or their module's test group.
456
457Tests marked (TP1) after name take --test-params including:
458
459 'vid=N;strip_vlan=bool;add_vlan=bool'
460
461Test List:
462""")
Rich Lane943be672012-10-04 19:20:16 -0700463 for (modname, (mod, tests)) in test_modules.items():
Dan Talayco7f8dba82012-04-12 12:58:52 -0700464 mod_count += 1
Rich Lane80efd652013-01-02 14:05:20 -0800465 desc = (mod.__doc__ or "No description").strip().split('\n')[0]
Stephen Finucane01588652014-05-15 20:03:12 +0100466 print(" Module %13s: %s" % (mod.__name__, desc))
467
Rich Lane943be672012-10-04 19:20:16 -0700468 for (testname, test) in tests.items():
Stephen Finucane01588652014-05-15 20:03:12 +0100469 desc = (test.__doc__ or "No description").strip().split('\n')[0]
470
Rich Lanee811e7b2013-01-03 13:36:54 -0800471 groups = set(test._groups) - set(["all", "standard", modname])
Rich Lane37f42112013-01-03 13:41:49 -0800472 all_groups.update(test._groups)
Rich Lanee811e7b2013-01-03 13:36:54 -0800473 if groups:
474 desc = "(%s) %s" % (",".join(groups), desc)
Rich Lane5a9a1922013-01-11 14:29:30 -0800475 if hasattr(test, "_versions"):
476 desc = "(%s) %s" % (",".join(sorted(test._versions)), desc)
Stephen Finucane01588652014-05-15 20:03:12 +0100477
Rich Lanee811e7b2013-01-03 13:36:54 -0800478 start_str = " %s%s %s:" % (test._nonstandard and "*" or " ",
479 test._disabled and "!" or " ",
480 testname)
Stephen Finucane01588652014-05-15 20:03:12 +0100481 print(" %22s : %s" % (start_str, desc))
Dan Talayco7f8dba82012-04-12 12:58:52 -0700482 test_count += 1
Dan Talayco79f36082010-03-11 16:53:53 -0800483 print
Stephen Finucane01588652014-05-15 20:03:12 +0100484 print("'%d' modules shown with a total of '%d' tests\n" %
485 (mod_count, test_count))
486 print("Test groups: %s" % (', '.join(sorted(all_groups))))
Rich Lane37f42112013-01-03 13:41:49 -0800487
Dan Talayco2c0dba32010-03-06 22:47:06 -0800488 sys.exit(0)
489
Rich Lane5a9a1922013-01-11 14:29:30 -0800490test_modules = prune_tests(test_specs, test_modules, config["openflow_version"])
Rich Laned8e45482013-01-02 17:36:21 -0800491
Jeffrey Townsend6614f972012-08-16 13:27:18 -0700492# Check if test list is requested; display and exit if so
493if config["list_test_names"]:
Rich Lane943be672012-10-04 19:20:16 -0700494 for (modname, (mod, tests)) in test_modules.items():
Rich Lane943be672012-10-04 19:20:16 -0700495 for (testname, test) in tests.items():
Stephen Finucane01588652014-05-15 20:03:12 +0100496 print("%s.%s" % (modname, testname))
497
Jeffrey Townsend6614f972012-08-16 13:27:18 -0700498 sys.exit(0)
499
Dan Talayco2c0dba32010-03-06 22:47:06 -0800500# Generate the test suite
501#@todo Decide if multiple suites are ever needed
502suite = unittest.TestSuite()
503
Rich Lane62b4fb12015-04-02 18:04:10 -0700504sorted_tests = []
Rich Lanec72ef462015-04-02 17:52:44 -0700505for (modname, (mod, tests)) in sorted(test_modules.items()):
506 for (testname, test) in sorted(tests.items()):
Rich Lane62b4fb12015-04-02 18:04:10 -0700507 sorted_tests.append(test)
508
509if config["random_order"]:
510 random.shuffle(sorted_tests)
511
512for test in sorted_tests:
513 suite.addTest(test())
Dan Talayco2c0dba32010-03-06 22:47:06 -0800514
Rich Lane51590f62012-10-09 15:06:29 -0700515# Allow platforms to import each other
516sys.path.append(config["platform_dir"])
517
Rich Lane8aebc5e2012-09-25 17:57:53 -0700518# Load the platform module
519platform_name = config["platform"]
520logging.info("Importing platform: " + platform_name)
521platform_mod = None
522try:
Rich Lane483e1542012-10-05 09:29:39 -0700523 platform_mod = imp.load_module(platform_name, *imp.find_module(platform_name, [config["platform_dir"]]))
Rich Lane8aebc5e2012-09-25 17:57:53 -0700524except:
525 logging.warn("Failed to import " + platform_name + " platform module")
526 raise
Dan Talayco48370102010-03-03 15:17:33 -0800527
528try:
Rich Lane8aebc5e2012-09-25 17:57:53 -0700529 platform_mod.platform_config_update(config)
Dan Talayco48370102010-03-03 15:17:33 -0800530except:
531 logging.warn("Could not run platform host configuration")
Rich Laneef403e52012-07-09 14:59:43 -0700532 raise
Dan Talayco48370102010-03-03 15:17:33 -0800533
534if not config["port_map"]:
Rich Lane8aebc5e2012-09-25 17:57:53 -0700535 die("Interface port map was not defined by the platform. Exiting.")
Dan Talayco48370102010-03-03 15:17:33 -0800536
537logging.debug("Configuration: " + str(config))
538logging.info("OF port map: " + str(config["port_map"]))
539
Rich Lanee55abf72012-07-26 20:11:42 -0700540oftest.ofutils.default_timeout = config["default_timeout"]
Rich Lane48f6aed2014-03-23 15:51:02 -0700541oftest.ofutils.default_negative_timeout = config["default_negative_timeout"]
Rich Laneda3b5ad2012-10-03 09:05:32 -0700542oftest.testutils.MINSIZE = config['minsize']
Rich Lanee55abf72012-07-26 20:11:42 -0700543
Rich Laneee57ad02012-07-13 15:40:36 -0700544if os.getuid() != 0 and not config["allow_user"]:
Stephen Finucane01588652014-05-15 20:03:12 +0100545 die("Super-user privileges required. Please re-run with sudo or as root.")
Rich Laned1d9c282012-10-04 22:07:10 -0700546 sys.exit(1)
Brandon Heller446c1432010-04-01 12:43:27 -0700547
Rich Lane8592bec2012-09-03 09:06:59 -0700548if config["random_seed"] is not None:
549 logging.info("Random seed: %d" % config["random_seed"])
550 random.seed(config["random_seed"])
Dan Talaycoec6084d2013-03-14 10:38:22 -0700551else:
552 # Generate random seed and report to log file
553 seed = random.randrange(100000000)
554 logging.info("Autogen random seed: %d" % seed)
555 random.seed(seed)
Rich Lane8592bec2012-09-03 09:06:59 -0700556
Rich Lane5bd6cf92012-10-04 17:57:24 -0700557# Remove python's signal handler which raises KeyboardError. Exiting from an
558# exception waits for all threads to terminate which might not happen.
559signal.signal(signal.SIGINT, signal.SIG_DFL)
Dan Talaycoac25cf32010-07-20 14:08:28 -0700560
561if __name__ == "__main__":
Stephen Finucane6219af72014-05-14 21:08:30 +0100562 profiler = profiler_setup(config)
Rich Lane9631f002014-03-21 18:05:16 -0700563
Rich Lane2c7812c2012-12-27 17:52:23 -0800564 # Set up the dataplane
565 oftest.dataplane_instance = oftest.dataplane.DataPlane(config)
Rich Lane472aaea2013-08-27 09:27:38 -0700566 pcap_setup(config)
Rich Lane2c7812c2012-12-27 17:52:23 -0800567 for of_port, ifname in config["port_map"].items():
568 oftest.dataplane_instance.port_add(ifname, of_port)
569
Dan Talaycoac25cf32010-07-20 14:08:28 -0700570 logging.info("*** TEST RUN START: " + time.asctime())
Stephen Finucanee016cf22014-04-16 22:04:11 +0100571 if config["xunit"]:
Stephen Finucane12782552014-05-20 21:55:04 +0100572 try:
573 import xmlrunner # fail-fast if module missing
574 except ImportError as ex:
575 oftest.dataplane_instance.kill()
576 profiler_teardown(profiler)
577 raise ex
Stephen Finucanee016cf22014-04-16 22:04:11 +0100578 runner = xmlrunner.XMLTestRunner(output=config["xunit_dir"],
579 outsuffix="",
580 verbosity=2)
581 result = runner.run(suite)
582 else:
583 result = unittest.TextTestRunner(verbosity=2).run(suite)
Rich Lane69fd8e02013-08-23 16:23:42 -0700584 oftest.open_logfile('main')
Rich Laneda3b5ad2012-10-03 09:05:32 -0700585 if oftest.testutils.skipped_test_count > 0:
Stephen Finucane01588652014-05-15 20:03:12 +0100586 message = "Skipped %d test(s)" % oftest.testutils.skipped_test_count
587 logging.info(message)
588 logging.info("*** TEST RUN END : %s", time.asctime())
Rich Lane2c7812c2012-12-27 17:52:23 -0800589
590 # Shutdown the dataplane
591 oftest.dataplane_instance.kill()
592 oftest.dataplane_instance = None
593
Stephen Finucane6219af72014-05-14 21:08:30 +0100594 profiler_teardown(profiler)
Rich Lane9631f002014-03-21 18:05:16 -0700595
Rich Lane50d42eb2012-07-16 11:57:03 -0700596 if result.failures or result.errors:
Shudong Zhoud895fcb2012-08-01 19:09:55 -0700597 # exit(1) hangs sometimes
598 os._exit(1)
Rich Laneda3b5ad2012-10-03 09:05:32 -0700599 if oftest.testutils.skipped_test_count > 0 and config["fail_skipped"]:
Shudong Zhoud895fcb2012-08-01 19:09:55 -0700600 os._exit(1)