From 257d3f847b06bbc3bed0fddde549e2ab7da13a05 Mon Sep 17 00:00:00 2001 From: Jay Pipes Date: Sun, 8 Jul 2012 23:01:31 -0400 Subject: [PATCH] Adds a script for tracking bug skips in tempest New file tools/skip_tracker.py can be used to show the status and priority of bugs that are marking test methods for skipping, and instruct the caller to remove skips on bugs that have been fixed in upstream. Output looks like this: jpipes@uberbox:~/repos/tempest$ python tools/skip_tracker.py INFO: Total bug skips found: 52 INFO: Total unique bugs causing skips: 30 INFO: Bug # 940500 ( Medium - Fix Released) INFO: Bug # 963248 ( Undecided - Invalid) INFO: Bug # 966249 ( Undecided - Fix Released) INFO: Bug # 987121 ( Medium - Fix Released) INFO: Bug # 988920 ( Undecided - Opinion) INFO: Bug # 997725 ( Medium - Fix Released) INFO: Bug # 999084 ( Medium - Triaged) INFO: Bug # 999209 ( Low - Fix Released) INFO: Bug # 999219 ( High - Triaged) INFO: Bug # 999567 ( Medium - Fix Released) INFO: Bug # 999594 ( Medium - In Progress) INFO: Bug # 999608 ( Low - Fix Released) INFO: Bug #1002892 ( Undecided - Invalid) INFO: Bug #1002901 ( Undecided - Invalid) INFO: Bug #1002911 ( Undecided - Invalid) INFO: Bug #1002918 ( Undecided - Invalid) INFO: Bug #1002924 ( Undecided - Incomplete) INFO: Bug #1002926 ( Undecided - Invalid) INFO: Bug #1002935 ( Undecided - Invalid) INFO: Bug #1004007 ( Low - Confirmed) INFO: Bug #1004564 ( Low - Confirmed) INFO: Bug #1005397 ( Undecided - Invalid) INFO: Bug #1005423 ( Low - Triaged) INFO: Bug #1006033 ( Undecided - New) INFO: Bug #1006725 ( Low - Triaged) INFO: Bug #1006857 ( Low - Confirmed) INFO: Bug #1006875 ( Low - Confirmed) INFO: Bug #1014647 ( Medium - Confirmed) INFO: Bug #1014683 ( Undecided - New) INFO: Bug #1022411 ( Undecided - In Progress) The following bugs have been fixed and the corresponding skips should be removed from the test cases: 940500 966249 987121 997725 999209 999567 999608 Change-Id: Ic58fc8beb2f6134504d4eb2f6ebe40fa24fe06f6 --- tools/skip_tracker.py | 105 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 tools/skip_tracker.py diff --git a/tools/skip_tracker.py b/tools/skip_tracker.py new file mode 100644 index 0000000000..fd15c1c01a --- /dev/null +++ b/tools/skip_tracker.py @@ -0,0 +1,105 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Track test skips via launchpadlib API and raise alerts if a bug +is fixed but a skip is still in the Tempest test code +""" + +import logging +import os +import re + +from launchpadlib import launchpad + +BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +TESTDIR = os.path.join(BASEDIR, 'tempest', 'tests') +LPCACHEDIR = os.path.expanduser('~/.launchpadlib/cache') + + +def info(msg, *args, **kwargs): + logging.info(msg, *args, **kwargs) + + +def debug(msg, *args, **kwargs): + logging.debug(msg, *args, **kwargs) + + +def find_skips(start=TESTDIR): + """ + Returns a list of tuples (method, bug) that represent + test methods that have been decorated to skip because of + a particular bug. + """ + results = [] + debug("Searching in %s", start) + for root, _dirs, files in os.walk(start): + for name in files: + if name.startswith('test_') and name.endswith('py'): + path = os.path.join(root, name) + debug("Searching in %s", path) + results += find_skips_in_file(path) + return results + + +def find_skips_in_file(path): + """ + Return the skip tuples in a test file + """ + BUG_RE = re.compile(r'.*skip\(.*[bB]ug\s*(\d+)') + DEF_RE = re.compile(r'.*def (\w+)\(') + bug_found = False + results = [] + lines = open(path, 'rb').readlines() + for x, line in enumerate(lines): + if not bug_found: + res = BUG_RE.match(line) + if res: + bug_no = int(res.group(1)) + debug("Found bug skip %s on line %d", bug_no, x + 1) + bug_found = True + else: + res = DEF_RE.match(line) + if res: + method = res.group(1) + debug("Found test method %s skips for bug %d", method, bug_no) + results.append((method, bug_no)) + bug_found = False + return results + + +if __name__ == '__main__': + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) + results = find_skips() + unique_bugs = sorted(set([bug for (method, bug) in results])) + unskips = [] + info("Total bug skips found: %d", len(results)) + info("Total unique bugs causing skips: %d", len(unique_bugs)) + lp = launchpad.Launchpad.login_anonymously('grabbing bugs', 'production', LPCACHEDIR) + for bug_no in unique_bugs: + bug = lp.bugs[bug_no] + for task in bug.bug_tasks: + info("Bug #%7s (%12s - %12s)", bug_no, task.importance, task.status) + if task.status in ('Fix Released', 'Fix Committed'): + unskips.append(bug_no) + + if unskips: + print "The following bugs have been fixed and the corresponding skips" + print "should be removed from the test cases:" + print + for bug in unskips: + print " %7s" % bug