blob: d89ef3a7ac821a255c8dbc89a1e83717a98ff50a [file] [log] [blame]
Dan Talayco48370102010-03-03 15:17:33 -08001#!/usr/bin/env python
2"""
3@package oft
4
5OpenFlow test framework top level script
6
Rich Laneea5060d2013-01-06 13:59:00 -08007This script is the entry point for running OpenFlow tests using the OFT
8framework. For usage information, see --help or the README.
Dan Talayco48370102010-03-03 15:17:33 -08009
Rich Laneea5060d2013-01-06 13:59:00 -080010To add a new command line option, edit both the config_default dictionary and
11the config_setup function. The option's result will end up in the global
12oftest.config dictionary.
Dan Talayco48370102010-03-03 15:17:33 -080013"""
14
15import sys
Rich Lane95f078b2013-01-06 13:24:58 -080016import optparse
Dan Talayco2c0dba32010-03-06 22:47:06 -080017from subprocess import Popen,PIPE
Dan Talayco48370102010-03-03 15:17:33 -080018import logging
19import unittest
Dan Talayco2c0dba32010-03-06 22:47:06 -080020import time
Brandon Heller446c1432010-04-01 12:43:27 -070021import os
Rich Lane6b452bc2012-07-09 16:52:21 -070022import imp
Rich Lane8592bec2012-09-03 09:06:59 -070023import random
Rich Lane5bd6cf92012-10-04 17:57:24 -070024import signal
Rich Lane943be672012-10-04 19:20:16 -070025import fnmatch
Rich Lane2d6d4822013-01-08 10:49:16 -080026import copy
Dan Talayco48370102010-03-03 15:17:33 -080027
Rich Lanefadf3452012-10-03 16:23:37 -070028root_dir = os.path.dirname(os.path.realpath(__file__))
29
30pydir = os.path.join(root_dir, 'src', 'python')
Rich Lane39878042012-07-09 14:45:35 -070031if os.path.exists(os.path.join(pydir, 'oftest')):
32 # Running from source tree
33 sys.path.insert(0, pydir)
34
Rich Lane477f4812012-10-04 22:49:00 -070035import oftest
36from oftest import config
Rich Lanee55abf72012-07-26 20:11:42 -070037import oftest.ofutils
Rich Lane95f078b2013-01-06 13:24:58 -080038import oftest.help_formatter
Rich Lane3f7098c2013-03-12 10:28:32 -070039import loxi
Dan Talaycoba3745c2010-07-21 21:51:08 -070040
Dan Talayco48370102010-03-03 15:17:33 -080041##@var DEBUG_LEVELS
42# Map from strings to debugging levels
43DEBUG_LEVELS = {
44 'debug' : logging.DEBUG,
45 'verbose' : logging.DEBUG,
46 'info' : logging.INFO,
47 'warning' : logging.WARNING,
48 'warn' : logging.WARNING,
49 'error' : logging.ERROR,
50 'critical' : logging.CRITICAL
51}
52
Dan Talayco48370102010-03-03 15:17:33 -080053##@var config_default
54# The default configuration dictionary for OFT
55config_default = {
Rich Lane95f078b2013-01-06 13:24:58 -080056 # Miscellaneous options
57 "list" : False,
58 "list_test_names" : False,
59 "allow_user" : False,
60
61 # Test selection options
Rich Lanec76b09a2013-01-02 16:53:22 -080062 "test_spec" : "",
Rich Lane1a08d232013-01-04 16:03:50 -080063 "test_file" : None,
Rich Lane74b13d12013-05-03 17:58:50 -070064 "test_dir" : None,
Rich Lane95f078b2013-01-06 13:24:58 -080065
66 # Switch connection options
67 "controller_host" : "0.0.0.0", # For passive bind
Rich Lane4d1f3eb2013-10-03 13:45:57 -070068 "controller_port" : 6653,
Rich Lane95f078b2013-01-06 13:24:58 -080069 "switch_ip" : None, # If not none, actively connect to switch
Rich Lane15f26322013-01-08 11:23:24 -080070 "platform" : "eth",
Rich Lane95f078b2013-01-06 13:24:58 -080071 "platform_args" : None,
72 "platform_dir" : os.path.join(root_dir, "platforms"),
Rich Lane2d6d4822013-01-08 10:49:16 -080073 "interfaces" : [],
Rich Lane9fd05682013-01-10 15:30:38 -080074 "openflow_version" : "1.0",
Rich Lane95f078b2013-01-06 13:24:58 -080075
76 # Logging options
Dan Talayco48370102010-03-03 15:17:33 -080077 "log_file" : "oft.log",
Rich Lane69fd8e02013-08-23 16:23:42 -070078 "log_dir" : None,
Rich Lane95f078b2013-01-06 13:24:58 -080079 "debug" : "verbose",
Rich Lane9631f002014-03-21 18:05:16 -070080 "profile" : False,
Stephen Finucane6219af72014-05-14 21:08:30 +010081 "profile_file" : "profile.out",
Stephen Finucanee016cf22014-04-16 22:04:11 +010082 "xunit" : False,
83 "xunit_dir" : "xunit",
Rich Lane95f078b2013-01-06 13:24:58 -080084
85 # Test behavior options
86 "relax" : False,
Dan Talaycod7c80d12012-04-03 15:20:57 -070087 "test_params" : "None",
Rich Lane9a84a4f2012-07-17 12:27:42 -070088 "fail_skipped" : False,
Rich Lane48f6aed2014-03-23 15:51:02 -070089 "default_timeout" : 2.0,
90 "default_negative_timeout" : 0.01,
Jeffrey Townsend5cb7ed32012-08-17 18:11:01 +000091 "minsize" : 0,
Rich Lane8592bec2012-09-03 09:06:59 -070092 "random_seed" : None,
Stephen Finucane92f7cf62014-03-13 15:08:11 +000093 "disable_ipv6" : False,
Rich Lane95f078b2013-01-06 13:24:58 -080094
95 # Other configuration
Rich Lane15f26322013-01-08 11:23:24 -080096 "port_map" : {},
Dan Talayco48370102010-03-03 15:17:33 -080097}
98
Rich Lane95f078b2013-01-06 13:24:58 -080099def config_setup():
Dan Talayco48370102010-03-03 15:17:33 -0800100 """
101 Set up the configuration including parsing the arguments
102
Dan Talayco48370102010-03-03 15:17:33 -0800103 @return A pair (config, args) where config is an config
104 object and args is any additional arguments from the command line
105 """
106
Rich Lane4113a582013-01-03 10:13:02 -0800107 usage = "usage: %prog [options] (test|group)..."
108
Rich Lane95f078b2013-01-06 13:24:58 -0800109 description = """\
Rich Lane4113a582013-01-03 10:13:02 -0800110OFTest is a framework and set of tests for validating OpenFlow switches.
111
112The default configuration assumes that an OpenFlow 1.0 switch is attempting to
Rich Lane4d1f3eb2013-10-03 13:45:57 -0700113connect to a controller on the machine running OFTest, port 6653. Additionally,
Rich Lane4113a582013-01-03 10:13:02 -0800114the interfaces veth1, veth3, veth5, and veth7 should be connected to the switch's
115dataplane.
116
117If no positional arguments are given then OFTest will run all tests that
118depend only on standard OpenFlow 1.0. Otherwise each positional argument
119is interpreted as either a test name or a test group name. The union of
120these will be executed. To see what groups each test belongs to use the
Rich Lane948f0302013-01-07 10:59:08 -0800121--list option. Tests and groups can be subtracted from the result by
122prefixing them with the '^' character.
Rich Lane4113a582013-01-03 10:13:02 -0800123"""
124
Rich Lane2d6d4822013-01-08 10:49:16 -0800125 # Parse --interface
126 def check_interface(option, opt, value):
127 try:
128 ofport, interface = value.split('@', 1)
129 ofport = int(ofport)
130 except ValueError:
131 raise optparse.OptionValueError("incorrect interface syntax (got %s, expected 'ofport@interface')" % repr(value))
132 return (ofport, interface)
133
134 class Option(optparse.Option):
135 TYPES = optparse.Option.TYPES + ("interface",)
136 TYPE_CHECKER = copy.copy(optparse.Option.TYPE_CHECKER)
137 TYPE_CHECKER["interface"] = check_interface
138
Rich Lane95f078b2013-01-06 13:24:58 -0800139 parser = optparse.OptionParser(version="%prog 0.1",
140 usage=usage,
141 description=description,
Rich Lane2d6d4822013-01-08 10:49:16 -0800142 formatter=oftest.help_formatter.HelpFormatter(),
143 option_class=Option)
Dan Talayco48370102010-03-03 15:17:33 -0800144
145 # Set up default values
Rich Lane95f078b2013-01-06 13:24:58 -0800146 parser.set_defaults(**config_default)
Dan Talayco48370102010-03-03 15:17:33 -0800147
Dan Talayco2c0dba32010-03-06 22:47:06 -0800148 parser.add_option("--list", action="store_true",
Brandon Heller824504e2010-04-01 12:21:37 -0700149 help="List all tests and exit")
Rich Lane95f078b2013-01-06 13:24:58 -0800150 parser.add_option("--list-test-names", action='store_true',
151 help="List test names matching the test spec and exit")
Rich Laneee57ad02012-07-13 15:40:36 -0700152 parser.add_option("--allow-user", action="store_true",
153 help="Proceed even if oftest is not run as root")
Rich Lane95f078b2013-01-06 13:24:58 -0800154
155 group = optparse.OptionGroup(parser, "Test selection options")
156 group.add_option("-T", "--test-spec", "--test-list", help="Tests to run, separated by commas")
157 group.add_option("-f", "--test-file", help="File of tests to run, one per line")
158 group.add_option("--test-dir", type="string", help="Directory containing tests")
159 parser.add_option_group(group)
160
161 group = optparse.OptionGroup(parser, "Switch connection options")
162 group.add_option("-H", "--host", dest="controller_host",
163 help="IP address to listen on (default %default)")
164 group.add_option("-p", "--port", dest="controller_port",
165 type="int", help="Port number to listen on (default %default)")
166 group.add_option("-S", "--switch-ip", dest="switch_ip",
167 help="If set, actively connect to this switch by IP")
168 group.add_option("-P", "--platform", help="Platform module name (default %default)")
169 group.add_option("-a", "--platform-args", help="Custom arguments for the platform")
170 group.add_option("--platform-dir", type="string", help="Directory containing platform modules")
Rich Lane2d6d4822013-01-08 10:49:16 -0800171 group.add_option("--interface", "-i", type="interface", dest="interfaces", metavar="INTERFACE", action="append",
172 help="Specify a OpenFlow port number and the dataplane interface to use. May be given multiple times. Example: 1@eth1")
Rich Lane50cfa502013-04-25 13:45:08 -0700173 group.add_option("--of-version", "-V", dest="openflow_version", choices=loxi.version_names.values(),
Rich Lane9fd05682013-01-10 15:30:38 -0800174 help="OpenFlow version to use")
Rich Lane95f078b2013-01-06 13:24:58 -0800175 parser.add_option_group(group)
176
177 group = optparse.OptionGroup(parser, "Logging options")
Rich Lane69fd8e02013-08-23 16:23:42 -0700178 group.add_option("--log-file", help="Name of log file (default %default)")
179 group.add_option("--log-dir", help="Name of log directory")
Rich Lane95f078b2013-01-06 13:24:58 -0800180 dbg_lvl_names = sorted(DEBUG_LEVELS.keys(), key=lambda x: DEBUG_LEVELS[x])
181 group.add_option("--debug", choices=dbg_lvl_names,
182 help="Debug lvl: debug, info, warning, error, critical (default %default)")
183 group.add_option("-v", "--verbose", action="store_const", dest="debug",
184 const="verbose", help="Shortcut for --debug=verbose")
185 group.add_option("-q", "--quiet", action="store_const", dest="debug",
186 const="warning", help="Shortcut for --debug=warning")
Stephen Finucane6219af72014-05-14 21:08:30 +0100187 group.add_option("--profile", action="store_true", help="Enable Python profiling")
188 group.add_option("--profile-file", help="Output file for Python profiler")
Stephen Finucanee016cf22014-04-16 22:04:11 +0100189 group.add_option("--xunit", action="store_true", help="Enable xUnit-formatted results")
190 group.add_option("--xunit-dir", help="Output directory for xUnit-formatted results")
Rich Lane95f078b2013-01-06 13:24:58 -0800191 parser.add_option_group(group)
192
193 group = optparse.OptionGroup(parser, "Test behavior options")
194 group.add_option("--relax", action="store_true",
195 help="Relax packet match checks allowing other packets")
196 test_params_help = """Set test parameters: key=val;... (see --list)
197 """
198 group.add_option("-t", "--test-params", help=test_params_help)
199 group.add_option("--fail-skipped", action="store_true",
Rich Lane9a84a4f2012-07-17 12:27:42 -0700200 help="Return failure if any test was skipped")
Rich Lane48f6aed2014-03-23 15:51:02 -0700201 group.add_option("--default-timeout", type=float,
Rich Lanee55abf72012-07-26 20:11:42 -0700202 help="Timeout in seconds for most operations")
Rich Lane48f6aed2014-03-23 15:51:02 -0700203 group.add_option("--default-negative-timeout", type=float,
204 help="Timeout in seconds for negative checks")
Rich Lane95f078b2013-01-06 13:24:58 -0800205 group.add_option("--minsize", type="int",
206 help="Minimum allowable packet size on the dataplane.")
207 group.add_option("--random-seed", type="int",
208 help="Random number generator seed")
Stephen Finucane92f7cf62014-03-13 15:08:11 +0000209 group.add_option("--disable-ipv6", action="store_true",
210 help="Disable IPv6 tests")
Rich Lane95f078b2013-01-06 13:24:58 -0800211 parser.add_option_group(group)
Jeffrey Townsend5cb7ed32012-08-17 18:11:01 +0000212
Dan Talayco48370102010-03-03 15:17:33 -0800213 # Might need this if other parsers want command line
214 # parser.allow_interspersed_args = False
215 (options, args) = parser.parse_args()
216
Rich Lane74b13d12013-05-03 17:58:50 -0700217 # If --test-dir wasn't given, pick one based on the OpenFlow version
218 if options.test_dir == None:
219 if options.openflow_version == "1.0":
220 options.test_dir = os.path.join(root_dir, "tests")
221 else:
222 options.test_dir = os.path.join(root_dir, "tests-" + options.openflow_version)
223
Rich Lane95f078b2013-01-06 13:24:58 -0800224 # Convert options from a Namespace to a plain dictionary
225 config = config_default.copy()
226 for key in config.keys():
227 config[key] = getattr(options, key)
Dan Talayco48370102010-03-03 15:17:33 -0800228
229 return (config, args)
230
231def logging_setup(config):
232 """
233 Set up logging based on config
234 """
Rich Lane69fd8e02013-08-23 16:23:42 -0700235
236 logging.getLogger().setLevel(DEBUG_LEVELS[config["debug"]])
237
238 if config["log_dir"] != None:
239 if os.path.exists(config["log_dir"]):
240 import shutil
241 shutil.rmtree(config["log_dir"])
242 os.makedirs(config["log_dir"])
243 else:
244 if os.path.exists(config["log_file"]):
245 os.remove(config["log_file"])
246
247 oftest.open_logfile('main')
Dan Talayco48370102010-03-03 15:17:33 -0800248
Stephen Finucanee016cf22014-04-16 22:04:11 +0100249def xunit_setup(config):
250 """
251 Set up xUnit output based on config
252 """
253
254 if not config["xunit"]:
255 return
256
Stephen Finucane6219af72014-05-14 21:08:30 +0100257 if os.path.exists(config["xunit_dir"]):
258 import shutil
259 shutil.rmtree(config["xunit_dir"])
260 os.makedirs(config["xunit_dir"])
Stephen Finucanee016cf22014-04-16 22:04:11 +0100261
Rich Lane472aaea2013-08-27 09:27:38 -0700262def pcap_setup(config):
263 """
264 Set up dataplane packet capturing based on config
265 """
266
267 if config["log_dir"] == None:
268 filename = os.path.splitext(config["log_file"])[0] + '.pcap'
269 oftest.dataplane_instance.start_pcap(filename)
270 else:
271 # start_pcap is called per-test in base_tests
272 pass
273
Stephen Finucane6219af72014-05-14 21:08:30 +0100274def profiler_setup(config):
275 """
276 Set up profiler based on config
277 """
278
279 if not config["profile"]:
280 return
281
282 import cProfile
283 profiler = cProfile.Profile()
284 profiler.enable()
285
286 return profiler
287
288def profiler_teardown(profiler):
289 """
290 Tear down profiler based on config
291 """
292
293 if not config["profile"]:
294 return
295
296 profiler.disable()
297 profiler.dump_stats(config["profile_file"])
298
Rich Lane472aaea2013-08-27 09:27:38 -0700299
Rich Lane943be672012-10-04 19:20:16 -0700300def load_test_modules(config):
301 """
302 Load tests from the test_dir directory.
Dan Talayco2c0dba32010-03-06 22:47:06 -0800303
Rich Lane943be672012-10-04 19:20:16 -0700304 Test cases are subclasses of unittest.TestCase
Dan Talayco2c0dba32010-03-06 22:47:06 -0800305
Rich Lanecc45b8e2013-01-02 15:55:02 -0800306 Also updates the _groups member to include "standard" and
307 module test groups if appropriate.
308
Dan Talayco2c0dba32010-03-06 22:47:06 -0800309 @param config The oft configuration dictionary
Rich Lane943be672012-10-04 19:20:16 -0700310 @returns A dictionary from test module names to tuples of
311 (module, dictionary from test names to test classes).
Dan Talayco2c0dba32010-03-06 22:47:06 -0800312 """
313
Rich Lane943be672012-10-04 19:20:16 -0700314 result = {}
Dan Talayco2c0dba32010-03-06 22:47:06 -0800315
Rich Lane943be672012-10-04 19:20:16 -0700316 for root, dirs, filenames in os.walk(config["test_dir"]):
317 # Iterate over each python file
318 for filename in fnmatch.filter(filenames, '[!.]*.py'):
319 modname = os.path.splitext(os.path.basename(filename))[0]
Dan Talayco2c0dba32010-03-06 22:47:06 -0800320
Rich Lane943be672012-10-04 19:20:16 -0700321 try:
322 if sys.modules.has_key(modname):
323 mod = sys.modules[modname]
324 else:
325 mod = imp.load_module(modname, *imp.find_module(modname, [root]))
326 except:
327 logging.warning("Could not import file " + filename)
328 raise
Rich Lane520e4152012-07-09 16:18:16 -0700329
Rich Lane943be672012-10-04 19:20:16 -0700330 # Find all testcases defined in the module
331 tests = dict((k, v) for (k, v) in mod.__dict__.items() if type(v) == type and
Rich Lane9cef2742013-07-16 13:27:00 -0700332 issubclass(v, unittest.TestCase) and
333 hasattr(v, "runTest"))
Rich Lane943be672012-10-04 19:20:16 -0700334 if tests:
Rich Lanecc45b8e2013-01-02 15:55:02 -0800335 for (testname, test) in tests.items():
336 # Set default annotation values
337 if not hasattr(test, "_groups"):
338 test._groups = []
339 if not hasattr(test, "_nonstandard"):
340 test._nonstandard = False
341 if not hasattr(test, "_disabled"):
342 test._disabled = False
343
344 # Put test in its module's test group
345 if not test._disabled:
346 test._groups.append(modname)
347
348 # Put test in the standard test group
349 if not test._disabled and not test._nonstandard:
350 test._groups.append("standard")
351 test._groups.append("all") # backwards compatibility
352
Rich Lane943be672012-10-04 19:20:16 -0700353 result[modname] = (mod, tests)
Rich Lane520e4152012-07-09 16:18:16 -0700354
Rich Lane943be672012-10-04 19:20:16 -0700355 return result
Dan Talayco2c0dba32010-03-06 22:47:06 -0800356
Rich Lane5a9a1922013-01-11 14:29:30 -0800357def prune_tests(test_specs, test_modules, version):
Rich Lane15f64de2012-10-04 21:25:57 -0700358 """
Rich Lane5a9a1922013-01-11 14:29:30 -0800359 Return tests matching the given test-specs and OpenFlow version
Rich Lanec76b09a2013-01-02 16:53:22 -0800360 @param test_specs A list of group names or test names.
Rich Lane5a9a1922013-01-11 14:29:30 -0800361 @param version An OpenFlow version (e.g. "1.0")
Rich Lane15f64de2012-10-04 21:25:57 -0700362 @param test_modules Same format as the output of load_test_modules.
363 @returns Same format as the output of load_test_modules.
364 """
365 result = {}
Rich Lanec76b09a2013-01-02 16:53:22 -0800366 for e in test_specs:
Rich Lane15f64de2012-10-04 21:25:57 -0700367 matched = False
Rich Lane948f0302013-01-07 10:59:08 -0800368
369 if e.startswith('^'):
370 negated = True
371 e = e[1:]
372 else:
373 negated = False
374
Rich Lane15f64de2012-10-04 21:25:57 -0700375 for (modname, (mod, tests)) in test_modules.items():
Rich Lanecc45b8e2013-01-02 15:55:02 -0800376 for (testname, test) in tests.items():
377 if e in test._groups or e == "%s.%s" % (modname, testname):
378 result.setdefault(modname, (mod, {}))
Rich Lane948f0302013-01-07 10:59:08 -0800379 if not negated:
Rich Lane5a9a1922013-01-11 14:29:30 -0800380 if not hasattr(test, "_versions") or version in test._versions:
381 result[modname][1][testname] = test
Rich Lane948f0302013-01-07 10:59:08 -0800382 else:
383 if modname in result and testname in result[modname][1]:
384 del result[modname][1][testname]
385 if not result[modname][1]:
386 del result[modname]
Rich Lanecc45b8e2013-01-02 15:55:02 -0800387 matched = True
Rich Lane948f0302013-01-07 10:59:08 -0800388
Rich Lane15f64de2012-10-04 21:25:57 -0700389 if not matched:
Rich Lanecc45b8e2013-01-02 15:55:02 -0800390 die("test-spec element %s did not match any tests" % e)
Rich Lane948f0302013-01-07 10:59:08 -0800391
Rich Lane15f64de2012-10-04 21:25:57 -0700392 return result
393
Dan Talayco2c0dba32010-03-06 22:47:06 -0800394def die(msg, exit_val=1):
395 print msg
396 logging.critical(msg)
397 sys.exit(exit_val)
398
Dan Talayco79f36082010-03-11 16:53:53 -0800399def _space_to(n, str):
400 """
401 Generate a string of spaces to achieve width n given string str
402 If length of str >= n, return one space
403 """
404 spaces = n - len(str)
405 if spaces > 0:
406 return " " * spaces
407 return " "
408
Dan Talayco48370102010-03-03 15:17:33 -0800409#
410# Main script
411#
412
Rich Lane477f4812012-10-04 22:49:00 -0700413# Setup global configuration
Rich Lane95f078b2013-01-06 13:24:58 -0800414(new_config, args) = config_setup()
Rich Lane477f4812012-10-04 22:49:00 -0700415oftest.config.update(new_config)
Dan Talayco48370102010-03-03 15:17:33 -0800416
Rich Laned7a85c42012-09-28 15:38:45 -0700417logging_setup(config)
Stephen Finucanee016cf22014-04-16 22:04:11 +0100418xunit_setup(config)
Rich Laned7a85c42012-09-28 15:38:45 -0700419logging.info("++++++++ " + time.asctime() + " ++++++++")
420
Rich Lane9fd05682013-01-10 15:30:38 -0800421# Pick an OpenFlow protocol module based on the configured version
Rich Lane50cfa502013-04-25 13:45:08 -0700422name_to_version = dict((v,k) for k, v in loxi.version_names.iteritems())
423sys.modules["ofp"] = loxi.protocol(name_to_version[config["openflow_version"]])
Rich Lane9fd05682013-01-10 15:30:38 -0800424
425# HACK: testutils.py imports controller.py, which needs the ofp module
426import oftest.testutils
427
Rich Lanee284b6b2012-10-03 09:19:58 -0700428# Allow tests to import each other
429sys.path.append(config["test_dir"])
430
Rich Lanec76b09a2013-01-02 16:53:22 -0800431test_specs = args
432if config["test_spec"] != "":
433 print >> sys.stderr, "WARNING: The --test-spec option is deprecated"
434 test_specs += config["test_spec"].split(',')
Rich Lane1a08d232013-01-04 16:03:50 -0800435if config["test_file"] != None:
436 with open(config["test_file"], 'r') as f:
437 for line in f:
438 line, _, _ = line.partition('#') # remove comments
439 line = line.strip()
440 if line:
441 test_specs.append(line)
Rich Lanec76b09a2013-01-02 16:53:22 -0800442if test_specs == []:
443 test_specs = ["standard"]
444
Rich Laned8e45482013-01-02 17:36:21 -0800445test_modules = load_test_modules(config)
Dan Talayco2c0dba32010-03-06 22:47:06 -0800446
447# Check if test list is requested; display and exit if so
448if config["list"]:
Dan Talayco7f8dba82012-04-12 12:58:52 -0700449 mod_count = 0
450 test_count = 0
Rich Lane37f42112013-01-03 13:41:49 -0800451 all_groups = set()
Rich Lanee811e7b2013-01-03 13:36:54 -0800452 print """\
453Tests are shown grouped by module. If a test is in any groups beyond "standard"
454and its module's group then they are shown in parentheses."""
455 print
456 print """\
457Tests marked with '*' are non-standard and may require vendor extensions or
458special switch configuration. These are not part of the "standard" test group."""
459 print
460 print """\
461Tests marked with '!' are disabled because they are experimental, special-purpose,
462or are too long to be run normally. These are not part of the "standard" test
463group or their module's test group."""
464 print
465 print "Tests marked (TP1) after name take --test-params including:"
466 print " 'vid=N;strip_vlan=bool;add_vlan=bool'"
Rich Lanee811e7b2013-01-03 13:36:54 -0800467 print
468 print "Test List:"
Rich Lane943be672012-10-04 19:20:16 -0700469 for (modname, (mod, tests)) in test_modules.items():
Dan Talayco7f8dba82012-04-12 12:58:52 -0700470 mod_count += 1
Rich Lane80efd652013-01-02 14:05:20 -0800471 desc = (mod.__doc__ or "No description").strip().split('\n')[0]
Dan Talayco79f36082010-03-11 16:53:53 -0800472 start_str = " Module " + mod.__name__ + ": "
473 print start_str + _space_to(22, start_str) + desc
Rich Lane943be672012-10-04 19:20:16 -0700474 for (testname, test) in tests.items():
Dan Talayco551befa2010-07-15 17:05:32 -0700475 try:
Rich Lane0f5b9c72013-01-02 14:05:20 -0800476 desc = (test.__doc__ or "").strip()
Dan Talayco551befa2010-07-15 17:05:32 -0700477 desc = desc.split('\n')[0]
478 except:
479 desc = "No description"
Rich Lanee811e7b2013-01-03 13:36:54 -0800480 groups = set(test._groups) - set(["all", "standard", modname])
Rich Lane37f42112013-01-03 13:41:49 -0800481 all_groups.update(test._groups)
Rich Lanee811e7b2013-01-03 13:36:54 -0800482 if groups:
483 desc = "(%s) %s" % (",".join(groups), desc)
Rich Lane5a9a1922013-01-11 14:29:30 -0800484 if hasattr(test, "_versions"):
485 desc = "(%s) %s" % (",".join(sorted(test._versions)), desc)
Rich Lanee811e7b2013-01-03 13:36:54 -0800486 start_str = " %s%s %s:" % (test._nonstandard and "*" or " ",
487 test._disabled and "!" or " ",
488 testname)
Dan Talayco551befa2010-07-15 17:05:32 -0700489 if len(start_str) > 22:
490 desc = "\n" + _space_to(22, "") + desc
Dan Talayco79f36082010-03-11 16:53:53 -0800491 print start_str + _space_to(22, start_str) + desc
Dan Talayco7f8dba82012-04-12 12:58:52 -0700492 test_count += 1
Dan Talayco79f36082010-03-11 16:53:53 -0800493 print
Rich Lane15f64de2012-10-04 21:25:57 -0700494 print "%d modules shown with a total of %d tests" % \
495 (mod_count, test_count)
496 print
Rich Lane37f42112013-01-03 13:41:49 -0800497 print "Test groups: %s" % (', '.join(sorted(all_groups)))
498
Dan Talayco2c0dba32010-03-06 22:47:06 -0800499 sys.exit(0)
500
Rich Lane5a9a1922013-01-11 14:29:30 -0800501test_modules = prune_tests(test_specs, test_modules, config["openflow_version"])
Rich Laned8e45482013-01-02 17:36:21 -0800502
Jeffrey Townsend6614f972012-08-16 13:27:18 -0700503# Check if test list is requested; display and exit if so
504if config["list_test_names"]:
Rich Lane943be672012-10-04 19:20:16 -0700505 for (modname, (mod, tests)) in test_modules.items():
Rich Lane943be672012-10-04 19:20:16 -0700506 for (testname, test) in tests.items():
Rich Lanecc45b8e2013-01-02 15:55:02 -0800507 print "%s.%s" % (modname, testname)
Jeffrey Townsend6614f972012-08-16 13:27:18 -0700508 sys.exit(0)
509
Dan Talayco2c0dba32010-03-06 22:47:06 -0800510# Generate the test suite
511#@todo Decide if multiple suites are ever needed
512suite = unittest.TestSuite()
513
Rich Lane15f64de2012-10-04 21:25:57 -0700514for (modname, (mod, tests)) in test_modules.items():
515 for (testname, test) in tests.items():
Rich Lanecc45b8e2013-01-02 15:55:02 -0800516 suite.addTest(test())
Dan Talayco2c0dba32010-03-06 22:47:06 -0800517
Rich Lane51590f62012-10-09 15:06:29 -0700518# Allow platforms to import each other
519sys.path.append(config["platform_dir"])
520
Rich Lane8aebc5e2012-09-25 17:57:53 -0700521# Load the platform module
522platform_name = config["platform"]
523logging.info("Importing platform: " + platform_name)
524platform_mod = None
525try:
Rich Lane483e1542012-10-05 09:29:39 -0700526 platform_mod = imp.load_module(platform_name, *imp.find_module(platform_name, [config["platform_dir"]]))
Rich Lane8aebc5e2012-09-25 17:57:53 -0700527except:
528 logging.warn("Failed to import " + platform_name + " platform module")
529 raise
Dan Talayco48370102010-03-03 15:17:33 -0800530
531try:
Rich Lane8aebc5e2012-09-25 17:57:53 -0700532 platform_mod.platform_config_update(config)
Dan Talayco48370102010-03-03 15:17:33 -0800533except:
534 logging.warn("Could not run platform host configuration")
Rich Laneef403e52012-07-09 14:59:43 -0700535 raise
Dan Talayco48370102010-03-03 15:17:33 -0800536
537if not config["port_map"]:
Rich Lane8aebc5e2012-09-25 17:57:53 -0700538 die("Interface port map was not defined by the platform. Exiting.")
Dan Talayco48370102010-03-03 15:17:33 -0800539
540logging.debug("Configuration: " + str(config))
541logging.info("OF port map: " + str(config["port_map"]))
542
Rich Lanee55abf72012-07-26 20:11:42 -0700543oftest.ofutils.default_timeout = config["default_timeout"]
Rich Lane48f6aed2014-03-23 15:51:02 -0700544oftest.ofutils.default_negative_timeout = config["default_negative_timeout"]
Rich Laneda3b5ad2012-10-03 09:05:32 -0700545oftest.testutils.MINSIZE = config['minsize']
Rich Lanee55abf72012-07-26 20:11:42 -0700546
Rich Laneee57ad02012-07-13 15:40:36 -0700547if os.getuid() != 0 and not config["allow_user"]:
Brandon Heller446c1432010-04-01 12:43:27 -0700548 print "ERROR: Super-user privileges required. Please re-run with " \
549 "sudo or as root."
Rich Laned1d9c282012-10-04 22:07:10 -0700550 sys.exit(1)
Brandon Heller446c1432010-04-01 12:43:27 -0700551
Rich Lane8592bec2012-09-03 09:06:59 -0700552if config["random_seed"] is not None:
553 logging.info("Random seed: %d" % config["random_seed"])
554 random.seed(config["random_seed"])
Dan Talaycoec6084d2013-03-14 10:38:22 -0700555else:
556 # Generate random seed and report to log file
557 seed = random.randrange(100000000)
558 logging.info("Autogen random seed: %d" % seed)
559 random.seed(seed)
Rich Lane8592bec2012-09-03 09:06:59 -0700560
Rich Lane5bd6cf92012-10-04 17:57:24 -0700561# Remove python's signal handler which raises KeyboardError. Exiting from an
562# exception waits for all threads to terminate which might not happen.
563signal.signal(signal.SIGINT, signal.SIG_DFL)
Dan Talaycoac25cf32010-07-20 14:08:28 -0700564
565if __name__ == "__main__":
Stephen Finucane6219af72014-05-14 21:08:30 +0100566 profiler = profiler_setup(config)
Rich Lane9631f002014-03-21 18:05:16 -0700567
Rich Lane2c7812c2012-12-27 17:52:23 -0800568 # Set up the dataplane
569 oftest.dataplane_instance = oftest.dataplane.DataPlane(config)
Rich Lane472aaea2013-08-27 09:27:38 -0700570 pcap_setup(config)
Rich Lane2c7812c2012-12-27 17:52:23 -0800571 for of_port, ifname in config["port_map"].items():
572 oftest.dataplane_instance.port_add(ifname, of_port)
573
Dan Talaycoac25cf32010-07-20 14:08:28 -0700574 logging.info("*** TEST RUN START: " + time.asctime())
Stephen Finucanee016cf22014-04-16 22:04:11 +0100575 if config["xunit"]:
Stephen Finucane12782552014-05-20 21:55:04 +0100576 try:
577 import xmlrunner # fail-fast if module missing
578 except ImportError as ex:
579 oftest.dataplane_instance.kill()
580 profiler_teardown(profiler)
581 raise ex
Stephen Finucanee016cf22014-04-16 22:04:11 +0100582 runner = xmlrunner.XMLTestRunner(output=config["xunit_dir"],
583 outsuffix="",
584 verbosity=2)
585 result = runner.run(suite)
586 else:
587 result = unittest.TextTestRunner(verbosity=2).run(suite)
Rich Lane69fd8e02013-08-23 16:23:42 -0700588 oftest.open_logfile('main')
Rich Laneda3b5ad2012-10-03 09:05:32 -0700589 if oftest.testutils.skipped_test_count > 0:
Dan Talaycoba3745c2010-07-21 21:51:08 -0700590 ts = " tests"
Stephen Finucane6219af72014-05-14 21:08:30 +0100591 if oftest.testutils.skipped_test_count == 1:
592 ts = " test"
Rich Laneda3b5ad2012-10-03 09:05:32 -0700593 logging.info("Skipped " + str(oftest.testutils.skipped_test_count) + ts)
594 print("Skipped " + str(oftest.testutils.skipped_test_count) + ts)
Dan Talaycoac25cf32010-07-20 14:08:28 -0700595 logging.info("*** TEST RUN END : " + time.asctime())
Rich Lane2c7812c2012-12-27 17:52:23 -0800596
597 # Shutdown the dataplane
598 oftest.dataplane_instance.kill()
599 oftest.dataplane_instance = None
600
Stephen Finucane6219af72014-05-14 21:08:30 +0100601 profiler_teardown(profiler)
Rich Lane9631f002014-03-21 18:05:16 -0700602
Rich Lane50d42eb2012-07-16 11:57:03 -0700603 if result.failures or result.errors:
Shudong Zhoud895fcb2012-08-01 19:09:55 -0700604 # exit(1) hangs sometimes
605 os._exit(1)
Rich Laneda3b5ad2012-10-03 09:05:32 -0700606 if oftest.testutils.skipped_test_count > 0 and config["fail_skipped"]:
Shudong Zhoud895fcb2012-08-01 19:09:55 -0700607 os._exit(1)