opendev
/
elastic-recheck
Code Issues Proposed changes

Merge "Use short build_uuids in elasticSearch queries"

This commit is contained in:
Jenkins 2014-01-24 17:03:17 +00:00 committed by Gerrit Code Review
parent 314486a36a 8c57a4f639
commit 3e4d066a30
4 changed files with 70 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
elastic_recheck/bot.py
View File

@ -145,7 +145,9 @@ class RecheckWatch(threading.Thread):
try: try:
event = stream.get_failed_tempest() event = stream.get_failed_tempest()
event.bugs = classifier.classify(event.change, event.rev) for short_build_uuid in event.short_build_uuids:
event.bugs |= set(classifier.classify(
event.change, event.rev, short_build_uuid))
if not event.bugs: if not event.bugs:
self._read(event) self._read(event)
else: else:

72
elastic_recheck/elasticRecheck.py
View File

@ -68,13 +68,20 @@ class FailEvent(object):
rev = None rev = None
project = None project = None
url = None url = None
bugs = [] bugs = set([])
short_build_uuids = []
comment = None
def __init__(self, event): def __init__(self, event):
self.change = event['change']['number'] self.change = event['change']['number']
self.rev = event['patchSet']['number'] self.rev = event['patchSet']['number']
self.project = event['change']['project'] self.project = event['change']['project']
self.url = event['change']['url'] self.url = event['change']['url']
self.comment = event["comment"]
self.bugs = set([])
def is_openstack_project(self):
return "tempest-dsvm-full" in self.comment
def name(self): def name(self):
return "%s,%s" % (self.change, self.rev) return "%s,%s" % (self.change, self.rev)
@ -120,22 +127,26 @@ class Stream(object):
for line in event['comment'].split("\n"): for line in event['comment'].split("\n"):
m = re.search("- ([\w-]+)\s*(http://\S+)\s*:\s*FAILURE", line) m = re.search("- ([\w-]+)\s*(http://\S+)\s*:\s*FAILURE", line)
if m: if m:
failed_tests[m.group(1)] = m.group(2) # The last 7 characters of the URL are the first 7 digits
# of the build_uuid.
failed_tests[m.group(1)] = {'url': m.group(2),
'short_build_uuid':
m.group(2)[-7:]}
return failed_tests return failed_tests
def _job_console_uploaded(self, change, patch, name): def _job_console_uploaded(self, change, patch, name, short_build_uuid):
query = qb.result_ready(change, patch, name) query = qb.result_ready(change, patch, name, short_build_uuid)
r = self.es.search(query, size='10') r = self.es.search(query, size='10')
if len(r) == 0: if len(r) == 0:
msg = ("Console logs not ready for %s %s,%s" % msg = ("Console logs not ready for %s %s,%s,%s" %
(name, change, patch)) (name, change, patch, short_build_uuid))
raise ConsoleNotReady(msg) raise ConsoleNotReady(msg)
else: else:
LOG.debug("Console ready for %s %s,%s" % LOG.debug("Console ready for %s %s,%s,%s" %
(name, change, patch)) (name, change, patch, short_build_uuid))
def _has_required_files(self, change, patch, name): def _has_required_files(self, change, patch, name, short_build_uuid):
query = qb.files_ready(change, patch) query = qb.files_ready(change, patch, name, short_build_uuid)
r = self.es.search(query, size='80') r = self.es.search(query, size='80')
files = [x['term'] for x in r.terms] files = [x['term'] for x in r.terms]
required = required_files(name) required = required_files(name)
@ -145,9 +156,6 @@ class Stream(object):
change, patch, name, missing_files)) change, patch, name, missing_files))
raise FilesNotReady(msg) raise FilesNotReady(msg)
def _is_openstack_project(self, event):
return "tempest-dsvm-full" in event["comment"]
def _does_es_have_data(self, change_number, patch_number, job_fails): def _does_es_have_data(self, change_number, patch_number, job_fails):
"""Wait till ElasticSearch is ready, but return False if timeout.""" """Wait till ElasticSearch is ready, but return False if timeout."""
NUMBER_OF_RETRIES = 20 NUMBER_OF_RETRIES = 20
@ -158,8 +166,12 @@ class Stream(object):
for i in range(NUMBER_OF_RETRIES): for i in range(NUMBER_OF_RETRIES):
try: try:
for job_name in job_fails: for job_name in job_fails:
#TODO(jogo) if there are three failed jobs and only the
#last one isn't ready we don't need to keep rechecking
# the first two
self._job_console_uploaded( self._job_console_uploaded(
change_number, patch_number, job_name) change_number, patch_number, job_name,
job_fails[job_name]['short_build_uuid'])
break break
except ConsoleNotReady as e: except ConsoleNotReady as e:
@ -177,8 +189,9 @@ class Stream(object):
if i == NUMBER_OF_RETRIES - 1: if i == NUMBER_OF_RETRIES - 1:
elapsed = datetime.datetime.now() - started_at elapsed = datetime.datetime.now() - started_at
msg = ("Console logs not available after %ss for %s %s,%s" % msg = ("Console logs not available after %ss for %s %s,%s,%s" %
(elapsed, job_name, change_number, patch_number)) (elapsed, job_name, change_number, patch_number,
job_fails[job_name]['short_build_uuid']))
raise ResultTimedOut(msg) raise ResultTimedOut(msg)
LOG.debug( LOG.debug(
@ -189,7 +202,8 @@ class Stream(object):
try: try:
for job_name in job_fails: for job_name in job_fails:
self._has_required_files( self._has_required_files(
change_number, patch_number, job_name) change_number, patch_number, job_name,
job_fails[job_name]['short_build_uuid'])
LOG.info( LOG.info(
"All files present for change_number: %s, patch_number: %s" "All files present for change_number: %s, patch_number: %s"
% (change_number, patch_number)) % (change_number, patch_number))
@ -200,8 +214,9 @@ class Stream(object):
# if we get to the end, we're broken # if we get to the end, we're broken
elapsed = datetime.datetime.now() - started_at elapsed = datetime.datetime.now() - started_at
msg = ("Required files not ready after %ss for %s %d,%d" % msg = ("Required files not ready after %ss for %s %d,%d,%s" %
(elapsed, job_name, change_number, patch_number)) (elapsed, job_name, change_number, patch_number,
job_fails[job_name]['short_build_uuid']))
raise ResultTimedOut(msg) raise ResultTimedOut(msg)
def get_failed_tempest(self): def get_failed_tempest(self):
@ -214,13 +229,16 @@ class Stream(object):
# nothing to see here, lets try the next event # nothing to see here, lets try the next event
continue continue
fevent = FailEvent(event)
# bail if it's not an openstack project # bail if it's not an openstack project
if not self._is_openstack_project(event): if not fevent.is_openstack_project():
continue continue
fevent = FailEvent(event)
LOG.info("Looking for failures in %s,%s on %s" % LOG.info("Looking for failures in %s,%s on %s" %
(fevent.change, fevent.rev, ", ".join(failed_jobs))) (fevent.change, fevent.rev, ", ".join(failed_jobs)))
fevent.short_build_uuids = [
v['short_build_uuid'] for v in failed_jobs.values()]
if self._does_es_have_data(fevent.change, fevent.rev, failed_jobs): if self._does_es_have_data(fevent.change, fevent.rev, failed_jobs):
return fevent return fevent
@ -267,7 +285,8 @@ class Classifier():
es_query = qb.generic(query, facet=facet) es_query = qb.generic(query, facet=facet)
return self.es.search(es_query, size=size) return self.es.search(es_query, size=size)
def classify(self, change_number, patch_number, skip_resolved=True): def classify(self, change_number, patch_number, short_build_uuid,
skip_resolved=True):
"""Returns either empty list or list with matched bugs.""" """Returns either empty list or list with matched bugs."""
LOG.debug("Entering classify") LOG.debug("Entering classify")
#Reload each time #Reload each time
@ -277,7 +296,8 @@ class Classifier():
LOG.debug( LOG.debug(
"Looking for bug: https://bugs.launchpad.net/bugs/%s" "Looking for bug: https://bugs.launchpad.net/bugs/%s"
% x['bug']) % x['bug'])
query = qb.single_patch(x['query'], change_number, patch_number) query = qb.single_patch(x['query'], change_number, patch_number,
short_build_uuid)
results = self.es.search(query, size='10') results = self.es.search(query, size='10')
if len(results) > 0: if len(results) > 0:
bug_matches.append(x['bug']) bug_matches.append(x['bug'])
@ -304,7 +324,11 @@ def main():
rev = event['patchSet']['number'] rev = event['patchSet']['number']
print "=======================" print "======================="
print "https://review.openstack.org/#/c/%(change)s/%(rev)s" % locals() print "https://review.openstack.org/#/c/%(change)s/%(rev)s" % locals()
bug_numbers = classifier.classify(change, rev) bug_numbers = []
for short_build_uuid in event.short_build_uuids:
bug_numbers = bug_numbers + classifier.classify(
change, rev, short_build_uuid)
bug_numbers = set(bug_numbers)
if not bug_numbers: if not bug_numbers:
print "unable to classify failure" print "unable to classify failure"
else: else:

21
elastic_recheck/query_builder.py
View File

@ -59,7 +59,7 @@ def generic(raw_query, facet=None):
return query return query
def result_ready(review=None, patch=None, name=None): def result_ready(review=None, patch=None, name=None, short_build_uuid=None):
"""A query to determine if we have a failure for a particular patch. """A query to determine if we have a failure for a particular patch.
This is looking for a particular FAILURE line in the console log, which This is looking for a particular FAILURE line in the console log, which
@ -70,11 +70,12 @@ def result_ready(review=None, patch=None, name=None):
'AND build_status:"FAILURE" ' 'AND build_status:"FAILURE" '
'AND build_change:"%s" ' 'AND build_change:"%s" '
'AND build_patchset:"%s" ' 'AND build_patchset:"%s" '
'AND build_name:"%s"' % 'AND build_name:"%s"'
(review, patch, name)) 'AND build_uuid:%s*' %
(review, patch, name, short_build_uuid))
def files_ready(review, patch): def files_ready(review, patch, name, short_build_uuid):
"""A facetted query to ensure all the required files exist. """A facetted query to ensure all the required files exist.
When changes are uploaded to elastic search there is a delay in When changes are uploaded to elastic search there is a delay in
@ -84,11 +85,14 @@ def files_ready(review, patch):
""" """
return generic('build_status:"FAILURE" ' return generic('build_status:"FAILURE" '
'AND build_change:"%s" ' 'AND build_change:"%s" '
'AND build_patchset:"%s"' % (review, patch), 'AND build_patchset:"%s"'
'AND build_name:"%s"'
'AND build_uuid:%s*' %
(review, patch, name, short_build_uuid),
facet='filename') facet='filename')
def single_patch(query, review, patch): def single_patch(query, review, patch, short_build_uuid):
"""A query for a single patch (review + revision). """A query for a single patch (review + revision).
This is used to narrow down a particular kind of failure found in a This is used to narrow down a particular kind of failure found in a
@ -96,5 +100,6 @@ def single_patch(query, review, patch):
""" """
return generic('%s ' return generic('%s '
'AND build_change:"%s" ' 'AND build_change:"%s" '
'AND build_patchset:"%s"' % 'AND build_patchset:"%s"'
(query, review, patch)) 'AND build_uuid:%s*' %
(query, review, patch, short_build_uuid))

8
elastic_recheck/tests/unit/test_stream.py
View File

@ -45,6 +45,7 @@ class TestStream(tests.TestCase):
self.assertEqual(event.project, "openstack/keystone") self.assertEqual(event.project, "openstack/keystone")
self.assertEqual(event.name(), "64749,6") self.assertEqual(event.name(), "64749,6")
self.assertEqual(event.url, "https://review.openstack.org/64749") self.assertEqual(event.url, "https://review.openstack.org/64749")
self.assertEqual(event.short_build_uuids, ["5dd41fe", "d3fd328"])
event = stream.get_failed_tempest() event = stream.get_failed_tempest()
self.assertEqual(event.change, "63078") self.assertEqual(event.change, "63078")
@ -52,6 +53,7 @@ class TestStream(tests.TestCase):
self.assertEqual(event.project, "openstack/horizon") self.assertEqual(event.project, "openstack/horizon")
self.assertEqual(event.name(), "63078,19") self.assertEqual(event.name(), "63078,19")
self.assertEqual(event.url, "https://review.openstack.org/63078") self.assertEqual(event.url, "https://review.openstack.org/63078")
self.assertEqual(event.short_build_uuids, ["ab07162"])
event = stream.get_failed_tempest() event = stream.get_failed_tempest()
self.assertEqual(event.change, "65361") self.assertEqual(event.change, "65361")
@ -59,6 +61,7 @@ class TestStream(tests.TestCase):
self.assertEqual(event.project, "openstack/requirements") self.assertEqual(event.project, "openstack/requirements")
self.assertEqual(event.name(), "65361,2") self.assertEqual(event.name(), "65361,2")
self.assertEqual(event.url, "https://review.openstack.org/65361") self.assertEqual(event.url, "https://review.openstack.org/65361")
self.assertEqual(event.short_build_uuids, ["8209fb4"])
self.assertRaises( self.assertRaises(
fg.GerritDone, fg.GerritDone,
@ -81,8 +84,9 @@ class TestStream(tests.TestCase):
self.assertIn('check-tempest-dsvm-neutron', jobs) self.assertIn('check-tempest-dsvm-neutron', jobs)
self.assertEqual(jobs['check-requirements-integration-dsvm'], self.assertEqual(jobs['check-requirements-integration-dsvm'],
"http://logs.openstack.org/31/64831/1/check/" {'url': "http://logs.openstack.org/31/64831/1/check/"
"check-requirements-integration-dsvm/135d0b4") "check-requirements-integration-dsvm/135d0b4",
'short_build_uuid': '135d0b4'})
self.assertNotIn('gate-requirements-pep8', jobs) self.assertNotIn('gate-requirements-pep8', jobs)
self.assertNotIn('gate-requirements-python27', jobs) self.assertNotIn('gate-requirements-python27', jobs)