110 lines
3.8 KiB
Python
110 lines
3.8 KiB
Python
"""Generate Elastic-Recheck compatible yaml files from human readable queries.
|
|
|
|
It takes 'pattern' and 'tags', both of which can be strings or lists,
|
|
from input file and forms an elastic recheck query. This query is written in a
|
|
file with filename same as the bug number if bug URL is provided
|
|
or else the id from input file.
|
|
|
|
input: src/data/queries.yml
|
|
output: output/elastic-recheck/<id/bug_no>.yaml
|
|
"""
|
|
import os
|
|
import shutil
|
|
|
|
import yaml
|
|
|
|
dir_path = os.path.dirname(os.path.realpath(__file__))
|
|
# Source and destination files
|
|
queries_src = os.path.join(dir_path, "data", "queries.yml")
|
|
elastic_recheck_dest_dir = os.path.join(
|
|
os.path.dirname(dir_path), "output", "elastic-recheck"
|
|
)
|
|
# Make sure dest dir for er is present and clean
|
|
if os.path.exists(elastic_recheck_dest_dir):
|
|
shutil.rmtree(elastic_recheck_dest_dir)
|
|
os.makedirs(elastic_recheck_dest_dir)
|
|
|
|
# Accepted misc ER parameters
|
|
er_params = ("build_name", "build_status", "tags", "build_branch", "voting")
|
|
|
|
|
|
def form_query_misc_params(q_dict, er_key):
|
|
"""Forms additional filters in the query with misc params."""
|
|
msg = ""
|
|
if isinstance(q_dict[er_key], str):
|
|
msg += " AND " + "(" + er_key + ':"' + q_dict[er_key] + '")'
|
|
elif isinstance(q_dict[er_key], list):
|
|
msg += (
|
|
" AND ("
|
|
+ " OR ".join(er_key + ':"' + tags + '"' for tags in q_dict[er_key])
|
|
+ ")"
|
|
)
|
|
return msg
|
|
|
|
|
|
def form_not_query(q_dict, er_key):
|
|
"""Forms filters with NOT."""
|
|
not_er_key = "not_" + er_key
|
|
msg = ""
|
|
if isinstance(q_dict[not_er_key], str):
|
|
msg += " AND NOT " + "(" + er_key + ':"' + q_dict[not_er_key] + '")'
|
|
elif isinstance(q_dict[er_key], list):
|
|
msg += (
|
|
" AND NOT ("
|
|
+ " OR ".join(er_key + ':"' + tags + '"' for tags in q_dict[not_er_key])
|
|
+ ")"
|
|
)
|
|
return msg
|
|
|
|
|
|
def form_logstash_query(q_dict, msg):
|
|
"""Forms the logstash query."""
|
|
if isinstance(q_dict["pattern"], str):
|
|
# example -> message:"java.io.IOException"
|
|
msg = 'message:"' + q_dict["pattern"] + '"'
|
|
elif isinstance(q_dict["pattern"], list):
|
|
msg = " OR ".join('message:"' + pattern + '"' for pattern in q_dict["pattern"])
|
|
# Other ER params
|
|
for er_param in er_params:
|
|
if er_param in query_dict:
|
|
msg += form_query_misc_params(query_dict, er_param)
|
|
if "not_" + er_param in query_dict:
|
|
msg += form_not_query(query_dict, er_param)
|
|
return msg
|
|
|
|
|
|
with open(queries_src) as in_file:
|
|
queries_list = yaml.load(in_file, Loader=yaml.BaseLoader)
|
|
|
|
for query_dict in queries_list["queries"]:
|
|
suppress_graph = False
|
|
suppress_notification = False
|
|
# default
|
|
message = ""
|
|
if "url" in query_dict:
|
|
out_filename = query_dict["url"].split("/")[-1] + ".yaml"
|
|
else:
|
|
out_filename = query_dict["id"] + ".yaml"
|
|
# If logstash is defined just copy it, else go through the fields and form logstash query
|
|
if "logstash" in query_dict:
|
|
message = query_dict["logstash"]
|
|
elif "pattern" not in query_dict:
|
|
continue
|
|
# Assuming "pattern" is always present if it needs to be shown in ER
|
|
else:
|
|
message = form_logstash_query(query_dict, message)
|
|
if "suppress-graph" in query_dict:
|
|
suppress_graph = bool(query_dict["suppress-graph"])
|
|
if "suppress-notification" in query_dict:
|
|
suppress_notification = bool(query_dict["suppress-notification"])
|
|
er_query = {
|
|
"query": message,
|
|
"suppress-graph": suppress_graph,
|
|
"suppress-notification": suppress_notification,
|
|
}
|
|
if "msg" in query_dict:
|
|
er_query["msg"] = query_dict["msg"]
|
|
|
|
with open(os.path.join(elastic_recheck_dest_dir, out_filename), "w") as out_file:
|
|
yaml.dump(er_query, out_file, default_flow_style=False, width=88)
|