Adding real time bug verification
This patch adds a script that will check all the bugs in the yaml skip list file which has a bugzilla or launchpad attached and verify if the bug still open or closed. If closed, it will remove from the skip list. This also can generate a report showing what is still open and what was already closed, as well as generate the skiplist in tempest/testr format. Change-Id: Iadd508184c350d074600316ad15f5274df87f01b
This commit is contained in:
Arx Cruz
Sagi Shnaidman
parent
c21bd4df57
commit
9ced8ecd72
|
|
@ -29,6 +29,7 @@ Role Variables
|
|||
* `tempest_exit_on_failure`: true/false - whether to exit from role with tempest exit code (default: true)
|
||||
* `tempestmail_config`: config.yaml - name of config file for tempestmail script
|
||||
* `tempest_track_resources`: true/false - whether to save the state of resources after tempest run (default: true)
|
||||
* `check_tempest_bugs`: true/false - Will check every bugzilla and launchpad bug in the yaml skip file
|
||||
|
||||
Skip tests file
|
||||
---------------
|
||||
|
|
@ -61,6 +62,15 @@ Example of skip file:
|
|||
reason: 'glance is not calling glance-manage db_load_metadefs'
|
||||
lp: 'https://bugs.launchpad.net/tripleo/+bug/1664995'
|
||||
|
||||
|
||||
Real time bug check
|
||||
-------------------
|
||||
|
||||
If check_tempest_bugs is set to true, a script will be called and will check
|
||||
in real time, all tests who has a bugzilla or a launchpad bug. This will
|
||||
generate a new skip file, removing all the bugs that were already closed but
|
||||
wasn't updated in the yaml skip file yet.
|
||||
|
||||
Dependencies
|
||||
------------
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ public_net_pool_end: "{{ floating_ip_cidr|nthhost(120) }}"
|
|||
public_net_gateway: "{{ floating_ip_cidr|nthhost(1) }}"
|
||||
tempest_log_file: 'tempest_output.log'
|
||||
test_regex: smoke
|
||||
check_tempest_bugs: false
|
||||
|
||||
public_net_name: public
|
||||
public_network_type: flat
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
[DEFAULT]
|
||||
test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
|
||||
OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
|
||||
OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-500} \
|
||||
OS_TEST_LOCK_PATH=${OS_TEST_LOCK_PATH:-${TMPDIR:-'/tmp'}} \
|
||||
${PYTHON:-python} -m subunit.run discover -t ${OS_TOP_LEVEL:-./} ${OS_TEST_PATH:-./tests} $LISTOPT $IDOPTION
|
||||
test_id_option=--load-list $IDFILE
|
||||
test_list_option=--list
|
||||
group_regex=([^\.]*\.)*
|
||||
|
|
@ -0,0 +1,206 @@
|
|||
import argparse
|
||||
import bugzilla
|
||||
import logging
|
||||
import sys
|
||||
import xmlrpclib
|
||||
import yaml
|
||||
|
||||
from launchpadlib.launchpad import Launchpad
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
OPEN = 1
|
||||
CLOSED = 2
|
||||
INVALID = 3
|
||||
|
||||
|
||||
class LaunchpadConnector(object):
|
||||
def __init__(self, cachedir='/tmp/.launchpadlib/cache/'):
|
||||
self.cachedir = cachedir
|
||||
self.lp = Launchpad.login_anonymously('Bugs', 'production', cachedir,
|
||||
version='devel')
|
||||
|
||||
def get_bug_status(self, bug_id):
|
||||
try:
|
||||
bug = self.lp.bugs[bug_id]
|
||||
# We are assuming that last task have the final status
|
||||
# And we cannot slice from the last
|
||||
task = bug.bug_tasks[len(bug.bug_tasks) - 1]
|
||||
if task:
|
||||
if task.status in ['Fix Released', 'Fix Committed', 'Invalid']:
|
||||
return CLOSED
|
||||
else:
|
||||
return OPEN
|
||||
except KeyError:
|
||||
LOG.error('Bug {} does not exist in launchpad'.format(bug_id))
|
||||
return INVALID
|
||||
|
||||
|
||||
class BugzillaConnector(object):
|
||||
def __init__(self, url='https://bugzilla.redhat.com/xmlrpc.cgi'):
|
||||
self.bugzilla = bugzilla.Bugzilla(url=url)
|
||||
|
||||
def get_bug_status(self, bug_id):
|
||||
try:
|
||||
bug = self.bugzilla.getbug(bug_id)
|
||||
if bug.status == 'CLOSED':
|
||||
return CLOSED
|
||||
else:
|
||||
return OPEN
|
||||
except xmlrpclib.Fault as err:
|
||||
# Fault code 102 means it's a private bug and we don't have
|
||||
# permission to see, so we can't confirm if it's closed
|
||||
if err.faultCode == 102:
|
||||
return OPEN
|
||||
LOG.error('Bug {} failed with fault code {}'.format(bug_id,
|
||||
err.faultCode))
|
||||
return INVALID
|
||||
|
||||
|
||||
class VerifyBug(object):
|
||||
def __init__(self):
|
||||
self.bugzilla = BugzillaConnector()
|
||||
self.launchpad = LaunchpadConnector()
|
||||
|
||||
def check_bug_status(self, url):
|
||||
connector = self._get_connector(url)
|
||||
bug_id = self._get_id_from_url(url)
|
||||
return connector.get_bug_status(bug_id)
|
||||
|
||||
def is_bug_open(self, url):
|
||||
status = self.check_bug_status(url)
|
||||
if status in [CLOSED, INVALID]:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
def _get_id_from_url(self, url):
|
||||
if 'launchpad' in url:
|
||||
# The format is https://bugs.launchpad.net/tripleo/+bug/1577769
|
||||
return int(url.split('/')[-1])
|
||||
elif 'bugzilla' in url:
|
||||
return int(url.split('=')[-1])
|
||||
|
||||
def _get_connector(self, url):
|
||||
if 'launchpad' in url:
|
||||
return self.launchpad
|
||||
elif 'bugzilla' in url:
|
||||
return self.bugzilla
|
||||
else:
|
||||
raise ValueError('Cannot find a connector for {}'.format(url))
|
||||
|
||||
|
||||
class BugVerifyCmd(object):
|
||||
def __init__(self):
|
||||
self.skipped_failures = []
|
||||
|
||||
def parse_arguments(self, args):
|
||||
parser = argparse.ArgumentParser(description='Bug verify')
|
||||
parser.add_argument('--skip-file', dest='skip_file',
|
||||
help='Load skip file', required=True)
|
||||
parser.add_argument('--output', action='store_true',
|
||||
help='Print the output')
|
||||
parser.add_argument('--format', dest='output_format',
|
||||
default='yaml', help='Output format',
|
||||
choices=['yaml', 'txt'])
|
||||
parser.add_argument('--to-file', dest='to_file',
|
||||
help='Save the skip list to a file')
|
||||
parser.add_argument('--report', dest='report', action='store_true',
|
||||
help='Shows report at the end')
|
||||
parser.add_argument('--debug', dest='debug', action='store_true',
|
||||
help='Enable debug')
|
||||
self.args = parser.parse_args(args)
|
||||
|
||||
def setup_logging(self):
|
||||
level = logging.DEBUG if self.args.debug else logging.INFO
|
||||
logging.basicConfig(level=level,
|
||||
format='%(asctime)s %(levelname)s %(name)s: '
|
||||
'%(message)s')
|
||||
|
||||
def load_skip_file(self):
|
||||
known_failures = []
|
||||
try:
|
||||
with open(self.args.skip_file) as f:
|
||||
skip = yaml.load(f)
|
||||
for t in skip.get('known_failures'):
|
||||
bug = {'test': t.get('test'), 'reason': t.get('reason')}
|
||||
if t.get('lp'):
|
||||
bug['lp'] = t.get('lp')
|
||||
if t.get('bz'):
|
||||
bug['bz'] = t.get('bz')
|
||||
known_failures.append(bug)
|
||||
except IOError:
|
||||
LOG.error('File not found {}'.format(self.args.skip_file))
|
||||
finally:
|
||||
return known_failures
|
||||
|
||||
def _print_yaml(self, known_failures):
|
||||
return yaml.dump({'known_failures': known_failures},
|
||||
default_flow_style=False,
|
||||
explicit_start=True)
|
||||
|
||||
def _print_txt(self, known_failures):
|
||||
output = ''
|
||||
for bug in known_failures:
|
||||
output += '# {}\n'.format(bug.get('reason'))
|
||||
output += '{}\n'.format(bug.get('test'))
|
||||
return output
|
||||
|
||||
def get_output(self, known_failures, output_format):
|
||||
output = ''
|
||||
if output_format == 'txt':
|
||||
output = self._print_txt(known_failures)
|
||||
elif output_format == 'yaml':
|
||||
output = self._print_yaml(known_failures)
|
||||
else:
|
||||
raise ValueError(
|
||||
'Output format not supported: {}'.format(output_format))
|
||||
return output
|
||||
|
||||
def print_output(self, known_failures, output_format):
|
||||
print self.get_output(known_failures, output_format)
|
||||
|
||||
def show_report(self):
|
||||
print 'Here\'s the original list:'
|
||||
self.print_output(self.original_failures, self.args.output_format)
|
||||
print '\n\n'
|
||||
print 'Here\'s the skipped list:'
|
||||
self.print_output(self.skipped_failures, self.args.output_format)
|
||||
|
||||
def save_output(self, known_failures, output_format):
|
||||
output = self.get_output(known_failures, output_format)
|
||||
f = open(self.args.to_file, 'w')
|
||||
f.write(output)
|
||||
f.close()
|
||||
|
||||
def run(self):
|
||||
known_failures = self.load_skip_file()
|
||||
self.original_failures = known_failures
|
||||
open_failures = []
|
||||
|
||||
v_bug = VerifyBug()
|
||||
for bug in known_failures:
|
||||
LOG.debug('Checking bug: {}'.format(bug))
|
||||
if not bug.get('lp') and not bug.get('bz'):
|
||||
open_failures.append(bug)
|
||||
continue
|
||||
bug_url = bug.get('lp') or bug.get('bz')
|
||||
if not v_bug.is_bug_open(bug_url):
|
||||
self.skipped_failures.append(bug)
|
||||
else:
|
||||
open_failures.append(bug)
|
||||
if self.args.output:
|
||||
self.print_output(open_failures, self.args.output_format)
|
||||
if self.args.to_file:
|
||||
self.save_output(open_failures, self.args.output_format)
|
||||
if self.args.report:
|
||||
self.show_report()
|
||||
|
||||
|
||||
def main():
|
||||
bvc = BugVerifyCmd()
|
||||
bvc.parse_arguments(sys.argv[1:])
|
||||
bvc.setup_logging()
|
||||
bvc.run()
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
launchpadlib==1.10.5
|
||||
python-bugzilla==2.1.0
|
||||
PyYAML==3.12
|
||||
simplejson==3.10.0
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
mock==2.0.0
|
||||
os-testr==0.8.2
|
||||
pbr==3.0.1
|
||||
testresources==2.0.1
|
||||
testtools==2.3.0
|
||||
unittest2==1.1.0
|
||||
|
|
@ -0,0 +1,204 @@
|
|||
import mock
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
import xmlrpclib
|
||||
|
||||
from bugcheck import BugVerifyCmd
|
||||
from bugcheck import BugzillaConnector
|
||||
from bugcheck import LaunchpadConnector
|
||||
from bugcheck import VerifyBug
|
||||
from bugcheck import OPEN
|
||||
from bugcheck import CLOSED
|
||||
from bugcheck import INVALID
|
||||
|
||||
|
||||
class TestLaunchpadConnector(unittest.TestCase):
|
||||
|
||||
@mock.patch('launchpadlib.launchpad.Launchpad.login_anonymously')
|
||||
def test_get_bug_status(self, launchpad_mock):
|
||||
lp_connector = LaunchpadConnector()
|
||||
|
||||
bugs = launchpad_mock.return_value.bugs
|
||||
bug_tasks = bugs.__getitem__().bug_tasks
|
||||
item = bug_tasks.__getitem__()
|
||||
|
||||
for status in ['Fix Released', 'Fix Committed', 'Invalid']:
|
||||
item.status = status
|
||||
self.assertEquals(lp_connector.get_bug_status(1693838), CLOSED)
|
||||
|
||||
item.status = 'No idea'
|
||||
self.assertEquals(lp_connector.get_bug_status(1693838), OPEN)
|
||||
|
||||
bugs.__getitem__.side_effect = KeyError()
|
||||
self.assertEquals(lp_connector.get_bug_status(1693838), INVALID)
|
||||
|
||||
|
||||
class TestBugzillaConnector(unittest.TestCase):
|
||||
@mock.patch('bugzilla.Bugzilla')
|
||||
def test_get_bug_status(self, bugzilla_mock):
|
||||
bz_connector = BugzillaConnector()
|
||||
bug = bugzilla_mock.return_value.getbug
|
||||
bug.return_value.status = 'CLOSED'
|
||||
self.assertEquals(bz_connector.get_bug_status(123), CLOSED)
|
||||
bz_status = ['ASSIGNED', 'NEEDINFO', 'NEW', 'REOPENED', 'RESOLVED',
|
||||
'UNCONFIRMED', 'VERIFIRED']
|
||||
for status in bz_status:
|
||||
bug.return_value.status = status
|
||||
self.assertEquals(bz_connector.get_bug_status(123), OPEN)
|
||||
|
||||
bug.side_effect = xmlrpclib.Fault(faultCode=102,
|
||||
faultString='Permission')
|
||||
self.assertEquals(bz_connector.get_bug_status(123), OPEN)
|
||||
bug.side_effect = xmlrpclib.Fault(faultCode=42,
|
||||
faultString='Other fault')
|
||||
self.assertEquals(bz_connector.get_bug_status(123), INVALID)
|
||||
|
||||
|
||||
class TestVerifyBug(unittest.TestCase):
|
||||
@mock.patch('launchpadlib.launchpad.Launchpad.login_anonymously')
|
||||
@mock.patch('bugzilla.Bugzilla')
|
||||
def setUp(self, bz_mock, lp_mock):
|
||||
self.v_bug = VerifyBug()
|
||||
|
||||
def test__get_id_from_url(self):
|
||||
self.assertEquals(self.v_bug._get_id_from_url(
|
||||
'https://bugs.launchpad.net/tripleo/+bug/1577769'), 1577769)
|
||||
self.assertEquals(self.v_bug._get_id_from_url(
|
||||
'https://bugzilla.redhat.com/show_bug.cgi?id=1380187'), 1380187)
|
||||
|
||||
def test__get_connector(self):
|
||||
self.assertIsInstance(self.v_bug._get_connector(
|
||||
'https://bugs.launchpad.net/tripleo/+bug/1577769'),
|
||||
LaunchpadConnector)
|
||||
self.assertIsInstance(self.v_bug._get_connector(
|
||||
'https://bugzilla.redhat.com/show_bug.cgi?id=1380187'),
|
||||
BugzillaConnector)
|
||||
self.assertRaises(ValueError, self.v_bug._get_connector,
|
||||
'https://review.openstack.org')
|
||||
|
||||
@mock.patch('bugcheck.VerifyBug.check_bug_status')
|
||||
def test_is_bug_open(self, bug_status_mock):
|
||||
for status in [CLOSED, INVALID]:
|
||||
bug_status_mock.return_value = status
|
||||
self.assertEquals(self.v_bug.is_bug_open(
|
||||
'https://bugzilla.redhat.com/show_bug.cgi?id=1380187'), False)
|
||||
|
||||
bug_status_mock.return_value = OPEN
|
||||
self.assertEquals(self.v_bug.is_bug_open(
|
||||
'https://bugzilla.redhat.com/show_bug.cgi?id=1380187'), True)
|
||||
|
||||
|
||||
class TestBugVerifyCmd(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.fd_file, self.tmp_file = tempfile.mkstemp()
|
||||
self._populate_skip_file()
|
||||
self.known_failures = [
|
||||
{'test': '.*test_external_network_visibility',
|
||||
'reason': 'Tempest test "external network visibility" fails',
|
||||
'lp': 'https://bugs.launchpad.net/tripleo/+bug/1577769'},
|
||||
{'test': 'tempest.api.data_processing',
|
||||
'reason': 'tempest.api.data_processing tests failing on newton',
|
||||
'bz': 'https://bugzilla.redhat.com/show_bug.cgi?id=1357667'},
|
||||
{'test': 'neutron.tests.tempest.api.test_revisions.TestRevisions',
|
||||
'reason': 'New test, need investigation'}]
|
||||
self.txt_output = ('# Tempest test "external network visibility" '
|
||||
'fails\n'
|
||||
'.*test_external_network_visibility\n'
|
||||
'# tempest.api.data_processing tests failing on '
|
||||
'newton\n'
|
||||
'tempest.api.data_processing\n'
|
||||
'# New test, need investigation\n'
|
||||
'neutron.tests.tempest.api.test_revisions.'
|
||||
'TestRevisions\n')
|
||||
self.yaml_output = ('---\nknown_failures:\n'
|
||||
'- lp: https://bugs.launchpad.net/tripleo/+bug/'
|
||||
'1577769\n'
|
||||
' reason: Tempest test "external network '
|
||||
'visibility" fails\n'
|
||||
' test: .*test_external_network_visibility\n'
|
||||
'- bz: https://bugzilla.redhat.com/show_bug.cgi'
|
||||
'?id=1357667\n'
|
||||
' reason: tempest.api.data_processing tests '
|
||||
'failing on newton\n'
|
||||
' test: tempest.api.data_processing\n'
|
||||
'- reason: New test, need investigation\n'
|
||||
' test: neutron.tests.tempest.api.test_'
|
||||
'revisions.TestRevisions\n')
|
||||
self.cmd = BugVerifyCmd()
|
||||
self.cmd.parse_arguments(['--skip-file', self.tmp_file])
|
||||
|
||||
def tearDown(self):
|
||||
os.close(self.fd_file)
|
||||
os.unlink(self.tmp_file)
|
||||
|
||||
def _populate_skip_file(self):
|
||||
content = '''
|
||||
known_failures:
|
||||
- test: '.*test_external_network_visibility'
|
||||
reason: 'Tempest test "external network visibility" fails'
|
||||
lp: 'https://bugs.launchpad.net/tripleo/+bug/1577769'
|
||||
- test: 'tempest.api.data_processing'
|
||||
reason: 'tempest.api.data_processing tests failing on newton'
|
||||
bz: 'https://bugzilla.redhat.com/show_bug.cgi?id=1357667'
|
||||
- test: 'neutron.tests.tempest.api.test_revisions.TestRevisions'
|
||||
reason: 'New test, need investigation'
|
||||
'''
|
||||
self.skip_file = open(self.tmp_file, 'w')
|
||||
self.skip_file.write(content)
|
||||
self.skip_file.close()
|
||||
|
||||
def test_load_skip_file(self):
|
||||
known_failures = self.cmd.load_skip_file()
|
||||
self.assertEquals(known_failures, self.known_failures)
|
||||
|
||||
def test__print_txt(self):
|
||||
output = self.cmd._print_txt(self.known_failures)
|
||||
self.assertEquals(output, self.txt_output)
|
||||
|
||||
def test__print_yaml(self):
|
||||
output = self.cmd._print_yaml(self.known_failures)
|
||||
self.assertEquals(output, self.yaml_output)
|
||||
|
||||
@mock.patch('bugcheck.BugVerifyCmd._print_txt')
|
||||
@mock.patch('bugcheck.BugVerifyCmd._print_yaml')
|
||||
def test_get_output(self, yaml_mock, txt_mock):
|
||||
self.cmd.get_output(self.known_failures, 'txt')
|
||||
self.cmd.get_output(self.known_failures, 'yaml')
|
||||
yaml_mock.assert_called_once()
|
||||
txt_mock.assert_called_once()
|
||||
self.assertRaises(ValueError,
|
||||
self.cmd.get_output, self.known_failures, 'xml')
|
||||
|
||||
def test_save_output(self):
|
||||
fd, tmp_f = tempfile.mkstemp()
|
||||
cmd = BugVerifyCmd()
|
||||
cmd.parse_arguments(['--skip-file', self.tmp_file, '--to-file', tmp_f])
|
||||
cmd.save_output(self.known_failures, 'txt')
|
||||
output = open(tmp_f, 'r').readlines()
|
||||
expected = ['# Tempest test "external network visibility" fails\n',
|
||||
'.*test_external_network_visibility\n',
|
||||
'# tempest.api.data_processing tests failing on newton\n',
|
||||
'tempest.api.data_processing\n',
|
||||
'# New test, need investigation\n',
|
||||
'neutron.tests.tempest.api.test_revisions.TestRevisions\n']
|
||||
self.assertEquals(output, expected)
|
||||
|
||||
cmd.save_output(self.known_failures, 'yaml')
|
||||
output = open(tmp_f, 'r').readlines()
|
||||
expected = ['---\n',
|
||||
'known_failures:\n',
|
||||
'- lp: https://bugs.launchpad.net/tripleo/+bug/1577769\n',
|
||||
' reason: Tempest test "external network visibility" '
|
||||
'fails\n',
|
||||
' test: .*test_external_network_visibility\n',
|
||||
'- bz: https://bugzilla.redhat.com/show_bug.cgi?'
|
||||
'id=1357667\n',
|
||||
' reason: tempest.api.data_processing tests failing on '
|
||||
'newton\n',
|
||||
' test: tempest.api.data_processing\n',
|
||||
'- reason: New test, need investigation\n',
|
||||
' test: neutron.tests.tempest.api.test_revisions.Tes'
|
||||
'tRevisions\n']
|
||||
self.assertEquals(output, expected)
|
||||
|
||||
|
|
@ -22,3 +22,36 @@
|
|||
dest: "{{ working_dir }}/{{ skip_file }}"
|
||||
mode: 0644
|
||||
when: skip_file_src != ''
|
||||
|
||||
- ignore_errors: true
|
||||
block:
|
||||
- name: Copying bugcheck files
|
||||
synchronize:
|
||||
src: bugcheck/
|
||||
dest: "{{ working_dir }}/bugcheck/"
|
||||
use_ssh_args: true
|
||||
|
||||
- name: Copying skip file
|
||||
synchronize:
|
||||
src: "vars/tempest_skip_{{ release }}.yml"
|
||||
dest: "{{ working_dir }}/bugcheck/"
|
||||
use_ssh_args: true
|
||||
|
||||
- name: Setting virtualenv
|
||||
shell: >
|
||||
virtualenv "{{ working_dir }}/bugcheck/.venv"
|
||||
|
||||
- name: Installing requirements
|
||||
pip:
|
||||
requirements: "{{ working_dir }}/bugcheck/requirements.txt"
|
||||
virtualenv: "{{ working_dir }}/bugcheck/.venv"
|
||||
|
||||
- name: Verifying bugs in bugzilla and launchpad and generating skip file
|
||||
shell: >
|
||||
source "{{ working_dir }}"/bugcheck/.venv/bin/activate;
|
||||
python bugcheck.py --skip-file "{{ working_dir }}/bugcheck/tempest_skip_{{ release }}.yml"
|
||||
--to-file "{{ working_dir }}/{{ skip_file }}" --format txt
|
||||
args:
|
||||
chdir: "{{ working_dir }}/bugcheck"
|
||||
ignore_errors: yes
|
||||
when: check_tempest_bugs|bool
|
||||
|
|
|
|||
Loading…
Reference in New Issue