diff --git a/doc/source/ostestr.rst b/doc/source/ostestr.rst index 7d8b1cf..71d3da8 100644 --- a/doc/source/ostestr.rst +++ b/doc/source/ostestr.rst @@ -30,9 +30,12 @@ Options Path to a whitelist file, this file contains a separate regex on each newline --regex REGEX, -r REGEX - A normal testr selection regex. If a blacklist file is - specified, the regex will be appended to the end of - the generated regex from that file + A normal testr selection regex. + + --black-regex BLACK_REGEX, -B BLACK_REGEX + Test rejection regex. If the test cases durring a + search opration matches, it will be removed from the + final test list. --pretty, -p Print pretty output from subunit-trace. This is mutually exclusive with --subunit @@ -109,14 +112,29 @@ covers which works with pdb. For example:: Test Selection -------------- -ostestr is designed to build on top of the test selection in testr. testr only -exposed a regex option to select tests. This equivalent is functionality is +ostestr intially designed to build on top of the test selection in testr. +testr only exposed a regex option to select tests. This functionality is exposed via the --regex option. For example:: $ ostestr --regex 'magic\.regex' This will do a straight passthrough of the provided regex to testr. -Additionally, ostestr allows you to specify a blacklist file to define a set +When ostestr is asked to do more complex test selection than a sinlge regex, +it will ask testr for a full list of tests than passing the filtered test list +back to testr. +ostestr allows you do to do simple test exclusion via apssing rejection/black regexp:: + + $ ostestr --back-regex 'slow_tests|bad_tests' + +ostestr also allow you to combine these argumants:: + + $ ostestr --regex ui\.interface --back-regexp 'slow_tests|bad_tests' + +Here first we selected all tests which matches to 'ui\.interface', +than we are dropping all test which matches +'slow_tests|bad_tests' from the final list. + +ostestr also allows you to specify a blacklist file to define a set of regexes to exclude. You can specify a blacklist file with the --blacklist_file/-b option, for example:: @@ -129,17 +147,19 @@ start of a comment on a line. For example:: ^regex1 # Excludes these tests .*regex2 # exclude those tests -Will generate a regex to pass to testr which will exclude both any tests +The regexp used in the blacklist File or passed as argument, will be used +to drop tests from the initial selection list. +Will generate a list which will exclude both any tests matching '^regex1' and '.*regex2'. If a blacklist file is used in conjunction -with the --regex option the regex specified with --regex will be appended to -the generated output from the --blacklist_file. Also it's worth noting that the +with the --regex option the regex specified with --regex will be used for the intial +test selection. Also it's worth noting that the regex test selection options can not be used in conjunction with the --no-discover or --pdb options described in the previous section. This is because the regex selection requires using testr under the covers to actually do the filtering, and those 2 options do not use testr. -The dual of the blacklist file is the whitelist file which works in the exact -same manner, except that instead of excluding regex matches it includes them. +The dual of the blacklist file is the whitelist file which altering the initial +test selection regex, by joining the white list elements by '|'. You can specify the path to the file with --whitelist_file/-w, for example:: $ ostestr --whitelist_file $path_to_file @@ -150,9 +170,7 @@ The format for the file is more or less identical to the blacklist file:: ^regex1 # Include these tests .*regex2 # include those tests -However, instead of excluding the matches it will include them. Note that a -blacklist file can not be used at the same time as a whitelist file, they -are mutually exclusive. +However, instead of excluding the matches it will include them. It's also worth noting that you can use the test list option to dry run any selection arguments you are using. You just need to use --list/-l with your diff --git a/os_testr/ostestr.py b/os_testr/ostestr.py index 24c9981..3a7a8fa 100755 --- a/os_testr/ostestr.py +++ b/os_testr/ostestr.py @@ -46,10 +46,7 @@ def get_parser(args): 'contains a separate regex on each newline.') group = parser.add_mutually_exclusive_group() group.add_argument('--regex', '-r', - help='A normal testr selection regex. If a blacklist ' - 'file is specified, the regex will be appended ' - 'to the end of the generated regex from that ' - 'file.') + help='A normal testr selection regex.') group.add_argument('--path', metavar='FILE_OR_DIRECTORY', help='A file name or directory of tests to run.') group.add_argument('--no-discover', '-n', metavar='TEST_ID', @@ -57,6 +54,14 @@ def get_parser(args): "discover and just execute the test specified. " "A file name may be used in place of a test " "name.") + parser.add_argument('--black-regex', '-B', + help='Test rejection regex. If a test cases name ' + 'matches on re.search() operation , ' + 'it will be removed from the final test list. ' + 'Effectively the black-regexp is added to ' + ' black regexp list, but you do need to edit a file. ' + 'The black filtering happens after the initial ' + ' white selection, which by default is everything.') pretty = parser.add_mutually_exclusive_group() pretty.add_argument('--pretty', '-p', dest='pretty', action='store_true', help='Print pretty output from subunit-trace. This is ' @@ -263,15 +268,22 @@ def main(): msg = "You can not use until_failure mode with pdb or no-discover" print(msg) exit(5) + + if ((opts.pdb or opts.no_discover) and (opts.black_regex)): + msg = "You can not use black-regex with pdb or no-discover" + print(msg) + exit(7) + if opts.path: regex = rb.path_to_regex(opts.path) else: regex = opts.regex - if opts.blacklist_file or opts.whitelist_file: + if opts.blacklist_file or opts.whitelist_file or opts.black_regex: list_of_tests = tlb.construct_list(opts.blacklist_file, opts.whitelist_file, regex, + opts.black_regex, opts.print_exclude) exit(_call_testr_with_list(opts, list_of_tests, others)) else: diff --git a/os_testr/testlist_builder.py b/os_testr/testlist_builder.py index d49fbfd..af05795 100644 --- a/os_testr/testlist_builder.py +++ b/os_testr/testlist_builder.py @@ -42,7 +42,8 @@ def print_skips(regex, message, test_list): print('\n') -def construct_list(blacklist_file, whitelist_file, regex, print_exclude): +def construct_list(blacklist_file, whitelist_file, regex, black_regex, + print_exclude): """Filters the discovered test cases :retrun: iterable of strings. The strings are full @@ -68,6 +69,14 @@ def construct_list(blacklist_file, whitelist_file, regex, print_exclude): else: black_data = None + if black_regex: + msg = "Skipped bacuse of regexp provided as a command line argument:" + record = (re.compile(black_regex), msg, []) + if black_data: + black_data.append(record) + else: + black_data = [record] + search_filter = re.compile(regex) # NOTE(afazekas): we do not want to pass a giant re diff --git a/os_testr/tests/testlist_builder.py b/os_testr/tests/testlist_builder.py index 90d6da4..5424feb 100644 --- a/os_testr/tests/testlist_builder.py +++ b/os_testr/tests/testlist_builder.py @@ -47,9 +47,24 @@ class TestConstructList(base.TestCase): test_lists = ['fake_test(scen)[tag,bar])', 'fake_test(scen)[egg,foo])'] with mock.patch('os_testr.regex_builder._get_test_list', return_value=test_lists): - result = list_builder.construct_list(None, None, 'foo', False) + result = list_builder.construct_list(None, + None, + 'foo', + None, + False) self.assertEqual(list(result), ['fake_test(scen)[egg,foo])']) + def test_simple_black_re(self): + test_lists = ['fake_test(scen)[tag,bar])', 'fake_test(scen)[egg,foo])'] + with mock.patch('os_testr.regex_builder._get_test_list', + return_value=test_lists): + result = list_builder.construct_list(None, + None, + None, + 'foo', + False) + self.assertEqual(list(result), ['fake_test(scen)[tag,bar])']) + def test_blacklist(self): black_list = [(re.compile('foo'), 'foo not liked', [])] test_lists = ['fake_test(scen)[tag,bar])', 'fake_test(scen)[egg,foo])'] @@ -60,6 +75,7 @@ class TestConstructList(base.TestCase): result = list_builder.construct_list('file', None, 'fake_test', + None, False) self.assertEqual(list(result), ['fake_test(scen)[tag,bar])']) @@ -74,6 +90,7 @@ class TestConstructList(base.TestCase): result = list_builder.construct_list(None, 'file', None, + None, False) self.assertEqual(set(result), set(('fake_test1[tg]', 'fake_test2[tg]'))) @@ -93,6 +110,7 @@ class TestConstructList(base.TestCase): result = list_builder.construct_list('black_file', 'white_file', 'foo', + None, False) self.assertEqual(set(result), set(('fake_test1[tg]', 'fake_test3[tg,foo]')))