opendev/elastic-recheck
Code Issues Proposed changes

Add support for Zuulv3-specific parameters in elastic-recheck

Browse Source 
This commit ensures elastic-recheck is able to support zuul v2 and v3
simultaneously:

- Add message queries based on v3 completion messages
- Include job-output.txt where console.html was expected

Change-Id: If3d7990d892a9698a112d9ff1dd5160998a4efe6
Depends-On: I7e34206d7968bf128e140468b9a222ecbce3a8f1
This commit is contained in:
David Moreau-Simard 2017-10-03 20:08:31 -04:00 committed by David Moreau-Simard
parent fe21639a1a
commit 97f6408b54
No known key found for this signature in database
GPG Key ID: 33A07694CBB71ECC
12 changed files with 52 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
elastic_recheck/cmd/check_success.py
View File

@ -24,6 +24,7 @@ import time
from launchpadlib import launchpad
import elastic_recheck.config as er_config
import elastic_recheck.elasticRecheck as er
import elastic_recheck.results as er_results
@ -51,17 +52,14 @@ def all_fails(classifier):
so we can figure out how good we are doing on total classification.
"""
all_fails = {}
query = ('filename:"console.html" '
'AND message:"Finished: FAILURE" '
'AND build_queue:"gate"')
results = classifier.hits_by_query(query, size=30000)
results = classifier.hits_by_query(er_config.ALL_FAILS_QUERY, size=30000)
facets = er_results.FacetSet()
facets.detect_facets(results, ["build_uuid"])
for build in facets:
for result in facets[build]:
# not perfect, but basically an attempt to show the integrated
# gate. Would be nice if there was a zuul attr for this in es.
if re.search("(^openstack/|devstack|grenade)", result.project):
if re.search(er_config.INCLUDED_PROJECTS_REGEX, result.project):
all_fails["%s.%s" % (build, result.build_name)] = False
return all_fails

10
elastic_recheck/cmd/uncategorized_fails.py
View File

@ -125,7 +125,10 @@ def all_fails(classifier, config=None):
if result.project in integrated_gate_projects:
name = result.build_name
timestamp = dp.parse(result.timestamp)
log = result.log_url.split("console.html")[0]
if 'console.html' in result.log_url:
log = result.log_url.split('console.html')[0]
elif 'job-output.txt' in result.log_url:
log = result.log_url.split('job-output.txt')[0]
integrated_fails["%s.%s" % (build, name)] = {
'log': log,
'timestamp': timestamp,
@ -137,7 +140,10 @@ def all_fails(classifier, config=None):
if re.search(config.included_projects_regex, result.project):
name = result.build_name
timestamp = dp.parse(result.timestamp)
log = result.log_url.split("console.html")[0]
if 'console.html' in result.log_url:
log = result.log_url.split('console.html')[0]
elif 'job-output.txt' in result.log_url:
log = result.log_url.split('job-output.txt')[0]
other_fails["%s.%s" % (build, name)] = {
'log': log,
'timestamp': timestamp,

15
elastic_recheck/config.py
View File

@ -47,11 +47,16 @@ EXCLUDED_JOBS_REGEX = re.compile('(' + '|'.join(EXCLUDED_JOBS) + ')')
INCLUDED_PROJECTS_REGEX = "(^openstack/|devstack|grenade)"
ALL_FAILS_QUERY = ('filename:"console.html" '
'AND (message:"Finished: FAILURE" '
'OR message:"[Zuul] Job complete, result: FAILURE") '
'AND build_queue:"gate" '
'AND voting:"1"')
# TODO(dmsimard): Revisit this query once Zuul v2 is no longer supported
# Let's value legibility over pep8 line width here...
ALL_FAILS_QUERY = (
'((filename:"job-output.txt" AND message:"POST-RUN END" AND message:"project-config/playbooks/base/post-ssh")' # flake8: noqa
' OR '
'(filename:"console.html" AND (message:"[Zuul] Job complete" OR message:"[SCP] Copying console log" OR message:"Grabbing consoleLog"))' # flake8: noqa
' AND build_status:"FAILURE"'
' AND build_queue:"gate"'
' AND voting:"1"'
)
UNCAT_MAX_SEARCH_SIZE = 30000

6
elastic_recheck/elasticRecheck.py
View File

@ -31,7 +31,7 @@ from elastic_recheck import results
def required_files(job):
files = ['console.html']
files = []
if re.match("tempest-dsvm", job):
files.extend([
'logs/screen-n-api.txt',
@ -246,9 +246,11 @@ class Stream(object):
query = qb.files_ready(change, patch, name, build_short_uuid)
r = self.es.search(query, size='80', recent=True)
files = [x['term'] for x in r.terms]
# TODO(dmsimard): Reliably differentiate zuul v2 and v3 jobs
required = required_files(name)
missing_files = [x for x in required if x not in files]
if len(missing_files) != 0:
if (len(missing_files) != 0 or
('console.html' not in files and 'job-output.txt' not in files)):
msg = ("%s missing for %s %s,%s,%s" % (
missing_files, name, change, patch, build_short_uuid))
raise FilesNotReady(msg)

32
elastic_recheck/query_builder.py
View File

@ -69,22 +69,30 @@ def single_queue(query, queue, facet=None):
(query, queue), facet=facet)
def result_ready(review, patch, name, build_short_uuid):
def result_ready(change, patchset, name, short_uuid):
"""A query to determine if we have a failure for a particular patch.
This is looking for a particular FAILURE line in the console log, which
lets us know that we've got results waiting that we need to process.
"""
return generic('filename:"console.html" AND '
'(message:"[SCP] Copying console log" '
'OR message:"Grabbing consoleLog" '
'OR message:"[Zuul] Job complete") '
'AND build_status:"FAILURE" '
'AND build_change:"%s" '
'AND build_patchset:"%s" '
'AND build_name:"%s" '
'AND build_short_uuid:%s' %
(review, patch, name, build_short_uuid))
# TODO(dmsimard): Revisit this query once Zuul v2 is no longer supported
# Let's value legibility over pep8 line width here...
query = (
'((filename:"job-output.txt" AND message:"POST-RUN END" AND message:"project-config/playbooks/base/post-ssh")' # flake8: noqa
' OR '
'(filename:"console.html" AND (message:"[Zuul] Job complete" OR message:"[SCP] Copying console log" OR message:"Grabbing consoleLog"))' # flake8: noqa
' AND build_status:"FAILURE"'
' AND build_change:"{change}"'
' AND build_patchset:"{patchset}"'
' AND build_name:"{name}"'
' AND build_short_uuid:"{short_uuid}"'
)
return generic(query.format(
change=change,
patchset=patchset,
name=name,
short_uuid=short_uuid
))
def files_ready(review, patch, name, build_short_uuid):
@ -119,7 +127,7 @@ def single_patch(query, review, patch, build_short_uuid):
def most_recent_event():
return generic(
'filename:console.html '
'(filename:"console.html" OR filename:"job-output.txt") '
'AND (build_queue:gate OR build_queue:check) '
'AND NOT tags:_grokparsefailure '
'AND NOT message:"%{logmessage}" ')

2
queries/1260311.yaml
View File

@ -1,4 +1,4 @@
query: >
message:"java.io.InterruptedIOException"
AND tags:"console.html"
AND (tags:"console.html" OR tags:"job-output.txt")
suppress-graph: true

2
queries/1260654.yaml
View File

@ -2,5 +2,5 @@ query: >
message:"java.io.IOException"
AND message:"Remote call on"
AND message:"failed"
AND tags:"console.html"
AND (tags:"console.html" OR tags:"job-output.txt")
suppress-graph: true

2
queries/1269940.yaml
View File

@ -1,4 +1,4 @@
query: >
message:"[EnvInject] - [ERROR] - SEVERE ERROR occurs:"
AND tags:"console.html"
AND (tags:"console.html" OR tags:"job-output.txt")
suppress-graph: true

2
queries/1270309.yaml
View File

@ -1,4 +1,4 @@
query: >
message:"java.lang.InterruptedException"
AND tags:"console.html"
AND (tags:"console.html" OR tags:"job-output.txt")
suppress-graph: true

2
queries/1384373.yaml
View File

@ -1,3 +1,3 @@
query: >
message:"fatal: Could not read from remote repository."
AND tags:"console.html"
AND (tags:"console.html" OR tags:"job-output.txt")

2
queries/1695970.yaml
View File

@ -1,3 +1,3 @@
query: >-
message:"mount: mounting /dev/sr0 on /mnt failed: Device or resource busy"
AND tags:"console.html"
AND (tags:"console.html" OR tags:"job-output.txt")

2
queries/1718990.yaml
View File

@ -1,3 +1,3 @@
query: >
message:"HTTP 500 curl 22 The requested URL returned error: 500 Internal Server Error" AND
filename:"console.html"
(filename:"console.html" OR filename:"job-output.txt")