42e3402806
as part of trying to simplify the core elasticRecheck, refactor the query creation into a separate set of query_builder routines. This takes away some of the duplication between the queries, and attempts to add documentation to the uses for each of them. add elasticRecheck fake pyelasticsearch testing build basic fixtures for unit testing that let us fake out the interaction to pyelasticsearch. This uses the json samples added for previous testing as the return results should an inbound query match one of the queries we know about. If the query is unknown to us, return an empty result set. Unit testing for both cases included going all the way from the top level Classifier class. Change-Id: I0d23b649274b31e8f281aaac588c4c6113a11a47
99 lines
3.3 KiB
Python
99 lines
3.3 KiB
Python
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""Query builder for pyelasticsearch
|
|
|
|
A serious of utility methods to build the kinds of queries that are needed
|
|
by elastic recheck to talk with elastic search.
|
|
"""
|
|
|
|
|
|
def generic(raw_query, facet=None):
|
|
"""Base query builder
|
|
|
|
Takes a raw_query string for elastic search. This is typically the same
|
|
content that you've typed into logstash to get to a unique result.
|
|
|
|
Optionally supports a facet, which is required for certain opperations,
|
|
like ensuring that all the expected log files for a job have been
|
|
uploaded.
|
|
"""
|
|
|
|
# they pyelasticsearch inputs are incredibly structured dictionaries
|
|
# so be it
|
|
query = {
|
|
"sort": {
|
|
"@timestamp": {"order": "desc"}
|
|
},
|
|
"query": {
|
|
"query_string": {
|
|
"query": raw_query
|
|
}
|
|
}
|
|
}
|
|
# if we have a facet, the query gets expanded
|
|
if facet:
|
|
data = dict(field=facet, size=200)
|
|
# yes, elasticsearch is odd, and the way to do multiple facets
|
|
# is to specify the plural key value
|
|
if type(facet) == list:
|
|
data = dict(fields=facet, size=200)
|
|
|
|
query['facets'] = {
|
|
"tag": {
|
|
"terms": data
|
|
}
|
|
}
|
|
|
|
return query
|
|
|
|
|
|
def result_ready(review=None, patch=None):
|
|
"""A query to determine if we have a failure for a particular patch.
|
|
|
|
This is looking for a particular FAILURE line in the console log, which
|
|
lets us know that we've got results waiting that we need to process.
|
|
"""
|
|
return generic('filename:"console.html" AND '
|
|
'(@message:"Finished: FAILURE" OR message:"Finished: FAILURE") '
|
|
'AND build_change:"%s" '
|
|
'AND build_patchset:"%s"' %
|
|
(review, patch))
|
|
|
|
|
|
def files_ready(review, patch):
|
|
"""A facetted query to ensure all the required files exist.
|
|
|
|
When changes are uploaded to elastic search there is a delay in
|
|
getting all the required log fixes into the system. This query returns
|
|
facets for the failure on the filename, which ensures that we've
|
|
found all the files in the system.
|
|
"""
|
|
return generic('build_status:"FAILURE" '
|
|
'AND build_change:"%s" '
|
|
'AND build_patchset:"%s"' % (review, patch),
|
|
facet='filename')
|
|
|
|
|
|
def single_patch(query, review, patch):
|
|
"""A query for a single patch (review + revision).
|
|
|
|
This is used to narrow down a particular kind of failure found in a
|
|
particular patch iteration.
|
|
"""
|
|
return generic('%s '
|
|
'AND build_change:"%s" '
|
|
'AND build_patchset:"%s"' %
|
|
(query, review, patch))
|