"""Generate Elastic-Recheck compatible yaml files from human readable queries. It takes 'pattern' and 'tags', both of which can be strings or lists, from input file and forms an elastic recheck query. This query is written in a file with filename same as the bug number if bug URL is provided or else the id from input file. input: src/data/queries.yml output: output/elastic-recheck/.yaml """ import os import shutil import yaml dir_path = os.path.dirname(os.path.realpath(__file__)) # Source and destination files queries_src = os.path.join(dir_path, "data", "queries.yml") elastic_recheck_dest_dir = os.path.join( os.path.dirname(dir_path), "output", "elastic-recheck" ) # Make sure dest dir for er is present and clean if os.path.exists(elastic_recheck_dest_dir): shutil.rmtree(elastic_recheck_dest_dir) os.makedirs(elastic_recheck_dest_dir) # Accepted misc ER parameters er_params = ("build_name", "build_status", "tags", "build_branch", "voting") def form_query_misc_params(q_dict, er_key): """Forms additional filters in the query with misc params.""" msg = "" if isinstance(q_dict[er_key], str): msg += " AND " + "(" + er_key + ':"' + q_dict[er_key] + '")' elif isinstance(q_dict[er_key], list): msg += ( " AND (" + " OR ".join(er_key + ':"' + tags + '"' for tags in q_dict[er_key]) + ")" ) return msg def form_not_query(q_dict, er_key): """Forms filters with NOT.""" not_er_key = "not_" + er_key msg = "" if isinstance(q_dict[not_er_key], str): msg += " AND NOT " + "(" + er_key + ':"' + q_dict[not_er_key] + '")' elif isinstance(q_dict[er_key], list): msg += ( " AND NOT (" + " OR ".join(er_key + ':"' + tags + '"' for tags in q_dict[not_er_key]) + ")" ) return msg def form_logstash_query(q_dict, msg): """Forms the logstash query.""" if isinstance(q_dict["pattern"], str): # example -> message:"java.io.IOException" msg = 'message:"' + q_dict["pattern"] + '"' elif isinstance(q_dict["pattern"], list): msg = " OR ".join('message:"' + pattern + '"' for pattern in q_dict["pattern"]) # Other ER params for er_param in er_params: if er_param in query_dict: msg += form_query_misc_params(query_dict, er_param) if "not_" + er_param in query_dict: msg += form_not_query(query_dict, er_param) return msg with open(queries_src) as in_file: queries_list = yaml.load(in_file, Loader=yaml.BaseLoader) for query_dict in queries_list["queries"]: suppress_graph = False suppress_notification = False # default message = "" if "url" in query_dict: out_filename = query_dict["url"].split("/")[-1] + ".yaml" else: out_filename = query_dict["id"] + ".yaml" # If logstash is defined just copy it, else go through the fields and form logstash query if "logstash" in query_dict: message = query_dict["logstash"] elif "pattern" not in query_dict: continue # Assuming "pattern" is always present if it needs to be shown in ER else: message = form_logstash_query(query_dict, message) if "suppress-graph" in query_dict: suppress_graph = bool(query_dict["suppress-graph"]) if "suppress-notification" in query_dict: suppress_notification = bool(query_dict["suppress-notification"]) er_query = { "query": message, "suppress-graph": suppress_graph, "suppress-notification": suppress_notification, } if "msg" in query_dict: er_query["msg"] = query_dict["msg"] with open(os.path.join(elastic_recheck_dest_dir, out_filename), "w") as out_file: yaml.dump(er_query, out_file, default_flow_style=False, width=88)