From 939ca8ac23e761e040cf69bf00f0f8adeea563c0 Mon Sep 17 00:00:00 2001 From: frenzyfriday Date: Fri, 16 Apr 2021 15:33:29 +0200 Subject: [PATCH] Adding script to convert queries to er format 1. The source is src/data/queries.yml. This is the human readable file updated by user. 2. ^ this is converted to files which ER can understand by er_converter.py 3. If there is a bug url in the query entry the corresponding ER file is named .yaml - else the file is named after the query id and shows up as "private bug" in ER dashboard. 4. As a test the playbook er.yml checks if all query files generated can be parsed by ER. 5. Added docstrings for er and sova converters and removed ER specific comments from sova converter. Depends-On: https://review.opendev.org/c/openstack/tripleo-ci-health-queries/+/780282/ Change-Id: Iea8e6a37b80a95b28ad6e09d041802b7e660d76b --- .gitignore | 3 + output/README.md | 7 + output/elastic-recheck/1260654.yaml | 3 + output/elastic-recheck/curl_re.yaml | 2 + .../overcloud_create_failed.yaml | 2 + playbooks/er.yml | 35 +++ requirements.in | 1 + requirements.txt | 261 +++++++++++++++++- src/er-converter.py | 58 ++++ src/sova-converter.py | 14 +- tox.ini | 3 + 11 files changed, 377 insertions(+), 12 deletions(-) create mode 100644 output/README.md create mode 100644 output/elastic-recheck/1260654.yaml create mode 100644 output/elastic-recheck/curl_re.yaml create mode 100644 output/elastic-recheck/overcloud_create_failed.yaml create mode 100644 playbooks/er.yml create mode 100644 src/er-converter.py diff --git a/.gitignore b/.gitignore index 3efd499..f55c025 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,6 @@ dmypy.json # IDE .idea/ + +# tox-report +report.html diff --git a/output/README.md b/output/README.md new file mode 100644 index 0000000..674b1d2 --- /dev/null +++ b/output/README.md @@ -0,0 +1,7 @@ +# Generated files + +All files from within `output/` folder built from other sources. Do not +attempt to modify them by hand. + +You should include the generated files with your CR but never to +edit them manually. diff --git a/output/elastic-recheck/1260654.yaml b/output/elastic-recheck/1260654.yaml new file mode 100644 index 0000000..8dfc112 --- /dev/null +++ b/output/elastic-recheck/1260654.yaml @@ -0,0 +1,3 @@ +query: message:"java.io.IOException" AND message:"Remote call on" AND message:"failed" AND + (tags:"console.html" OR tags:"job-output.txt") +suppress-graph: true diff --git a/output/elastic-recheck/curl_re.yaml b/output/elastic-recheck/curl_re.yaml new file mode 100644 index 0000000..393dfdb --- /dev/null +++ b/output/elastic-recheck/curl_re.yaml @@ -0,0 +1,2 @@ +query: message:"couldn't open file" +suppress-graph: false diff --git a/output/elastic-recheck/overcloud_create_failed.yaml b/output/elastic-recheck/overcloud_create_failed.yaml new file mode 100644 index 0000000..740d6c0 --- /dev/null +++ b/output/elastic-recheck/overcloud_create_failed.yaml @@ -0,0 +1,2 @@ +query: message:"Stack overcloud CREATE_FAILED" +suppress-graph: false diff --git a/playbooks/er.yml b/playbooks/er.yml new file mode 100644 index 0000000..7b85a15 --- /dev/null +++ b/playbooks/er.yml @@ -0,0 +1,35 @@ +--- +- name: Validate that ER can parse what we produce + hosts: localhost + connection: local + gather_facts: true + + vars: + er_dir: '../output/elastic-recheck' + test_results_dir: '../test_results' + + tasks: + - name: Make sure dir for test output exists + file: + path: "{{ test_results_dir }}" + state: directory + mode: 0755 + + - name: Get list of er files from er directory + find: + paths: "{{ er_dir }}" + register: er_files_output + + - name: Run elastic_recheck_query for all er files + vars: + output_file: "{{ test_results_dir }}/{{ item.path | basename }}.log" + shell: | + set -e + elastic-recheck-query "{{ item.path }}" > "{{ output_file }}" 2>&1 + ! grep -q "pyelasticsearch.exceptions.ElasticHttpError" "{{ output_file }}" + # This also fails if an exception is found inside the output files + loop: "{{ er_files_output.files }}" + loop_control: + label: "{{ item.path | basename }}" + register: er_output + changed_when: false diff --git a/requirements.in b/requirements.in index 30ce364..e15471e 100644 --- a/requirements.in +++ b/requirements.in @@ -3,3 +3,4 @@ pydantic>=1.7.4 # MIT yq # Apache jsonschema # MIT PyYAML>=5.4.1 +git+https://opendev.org/opendev/elastic-recheck.git#egg=elastic-recheck diff --git a/requirements.txt b/requirements.txt index 25a450b..c103b08 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,47 +4,294 @@ # # pip-compile --output-file=requirements.txt requirements.in # +alembic==1.5.8 + # via + # oslo.db + # subunit2sql ansible-base==2.10.8 # via -r requirements.in argcomplete==1.12.2 # via yq +argparse==1.4.0 + # via unittest2 attrs==20.3.0 # via jsonschema +babel==2.9.0 + # via elastic-recheck +bcrypt==3.2.0 + # via paramiko +certifi==2020.12.5 + # via requests cffi==1.14.5 - # via cryptography + # via + # bcrypt + # cryptography + # pynacl +chardet==4.0.0 + # via requests cryptography==3.4.7 - # via ansible-base + # via + # ansible-base + # paramiko +debtcollector==2.2.0 + # via + # oslo.config + # oslo.db + # oslo.utils +decorator==5.0.7 + # via sqlalchemy-migrate +distro==1.5.0 + # via lazr.restfulclient +docutils==0.17.1 + # via python-daemon +git+https://opendev.org/opendev/elastic-recheck.git#egg=elastic-recheck + # via -r requirements.in +extras==1.0.0 + # via + # python-subunit + # testtools +fixtures==3.0.0 + # via testtools +gerritlib==0.10.0 + # via elastic-recheck +greenlet==1.0.0 + # via sqlalchemy +httplib2==0.19.1 + # via + # elastic-recheck + # launchpadlib + # lazr.restfulclient +idna==2.10 + # via requests +importlib-metadata==4.0.1 + # via keyring +irc==19.0.1 + # via elastic-recheck +iso8601==0.1.14 + # via oslo.utils +jaraco.classes==3.2.1 + # via jaraco.collections +jaraco.collections==3.3.0 + # via irc +jaraco.functools==3.3.0 + # via + # irc + # jaraco.text + # tempora +jaraco.logging==3.1.0 + # via irc +jaraco.stream==3.0.2 + # via irc +jaraco.text==3.5.0 + # via + # irc + # jaraco.collections jinja2==2.11.3 - # via ansible-base + # via + # ansible-base + # elastic-recheck jsonschema==3.2.0 # via -r requirements.in +keyring==23.0.1 + # via launchpadlib +launchpadlib==1.10.13 + # via elastic-recheck +lazr.restfulclient==0.14.3 + # via + # elastic-recheck + # launchpadlib +lazr.uri==1.0.5 + # via + # launchpadlib + # wadllib +linecache2==1.0.0 + # via traceback2 +lockfile==0.12.2 + # via + # elastic-recheck + # python-daemon +mako==1.1.4 + # via alembic markupsafe==1.1.1 - # via jinja2 + # via + # jinja2 + # mako +more-itertools==8.7.0 + # via + # irc + # jaraco.classes + # jaraco.functools +netaddr==0.8.0 + # via + # oslo.config + # oslo.utils +netifaces==0.10.9 + # via oslo.utils +oauthlib==3.1.0 + # via lazr.restfulclient +oslo.config==8.5.0 + # via + # oslo.db + # subunit2sql +oslo.db==8.5.0 + # via subunit2sql +oslo.i18n==5.0.1 + # via + # oslo.config + # oslo.db + # oslo.utils +oslo.utils==4.8.0 + # via oslo.db packaging==20.9 - # via ansible-base + # via + # ansible-base + # oslo.utils +paramiko==2.7.2 + # via gerritlib +pbr==5.5.1 + # via + # debtcollector + # elastic-recheck + # fixtures + # gerritlib + # oslo.db + # oslo.i18n + # oslo.utils + # sqlalchemy-migrate + # stevedore + # subunit2sql + # testresources + # testscenarios + # testtools pycparser==2.20 # via cffi pydantic==1.8.1 # via -r requirements.in +pyelasticsearch==0.7.1 + # via elastic-recheck +pymysql==1.0.2 + # via elastic-recheck +pynacl==1.4.0 + # via paramiko pyparsing==2.4.7 - # via packaging + # via + # httplib2 + # oslo.utils + # packaging pyrsistent==0.17.3 # via jsonschema +python-daemon==2.3.0 + # via elastic-recheck +python-dateutil==2.8.1 + # via + # alembic + # elastic-recheck + # subunit2sql +python-editor==1.0.4 + # via alembic +python-mimeparse==1.6.0 + # via testtools +python-subunit==1.4.0 + # via subunit2sql +pytz==2021.1 + # via + # babel + # elastic-recheck + # irc + # oslo.utils + # tempora pyyaml==5.4.1 # via # -r requirements.in # ansible-base + # elastic-recheck + # oslo.config # yq +requests==2.25.1 + # via + # elastic-recheck + # oslo.config + # pyelasticsearch +rfc3986==1.4.0 + # via oslo.config +simplejson==3.17.2 + # via pyelasticsearch six==1.15.0 - # via jsonschema + # via + # bcrypt + # debtcollector + # fixtures + # gerritlib + # jsonschema + # launchpadlib + # lazr.restfulclient + # oslo.i18n + # pyelasticsearch + # pynacl + # python-dateutil + # sqlalchemy-migrate + # subunit2sql + # testtools + # unittest2 +sqlalchemy-migrate==0.13.0 + # via oslo.db +sqlalchemy==1.4.11 + # via + # alembic + # elastic-recheck + # oslo.db + # sqlalchemy-migrate + # subunit2sql +sqlparse==0.4.1 + # via sqlalchemy-migrate +stevedore==3.3.0 + # via + # oslo.config + # oslo.db + # subunit2sql +subunit2sql==1.10.0 + # via elastic-recheck +tempita==0.5.2 + # via sqlalchemy-migrate +tempora==4.0.2 + # via + # irc + # jaraco.logging +testresources==2.0.1 + # via + # launchpadlib + # oslo.db +testscenarios==0.5.0 + # via oslo.db +testtools==2.4.0 + # via + # fixtures + # python-subunit + # testscenarios toml==0.10.2 # via yq +traceback2==1.4.0 + # via + # testtools + # unittest2 typing-extensions==3.7.4.3 # via pydantic +unittest2==1.1.0 + # via testtools +urllib3==1.26.4 + # via requests +wadllib==1.3.5 + # via + # launchpadlib + # lazr.restfulclient +wrapt==1.12.1 + # via debtcollector xmltodict==0.12.0 # via yq yq==2.12.0 # via -r requirements.in +zipp==3.4.1 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/src/er-converter.py b/src/er-converter.py new file mode 100644 index 0000000..6cb9ce8 --- /dev/null +++ b/src/er-converter.py @@ -0,0 +1,58 @@ +import json +import os +import yaml +import re + +""" +This script generates Elastic-Recheck compatible yaml files from human readable queries. +It takes 'pattern' and 'tags', both of which can be strings or lists, from input file and forms +an elastic recheck query. This query is written in a file with filename same as the bug number if bug URL is provided +or else the id from input file. + +input: src/data/queries.yml +output: output/elastic-recheck/.yaml +""" + +dir_path = os.path.dirname(os.path.realpath(__file__)) +# Source and destination files +queries_src = os.path.join(dir_path, 'data', 'queries.yml') +elastic_recheck_dest_dir = os.path.join(os.path.dirname(dir_path), 'output', 'elastic-recheck') +# Make sure dest dir for er is present +if not os.path.exists(elastic_recheck_dest_dir): + os.makedirs(elastic_recheck_dest_dir) + +with open(queries_src) as in_file: + queries_list = yaml.load(in_file, Loader=yaml.BaseLoader) + +for query_dict in queries_list['queries']: + if "pattern" not in query_dict: + continue + # Assuming "pattern" is always present if it needs to be shown in ER + suppress_graph = False # default + message = '' + if "url" in query_dict: + out_filename = query_dict["url"].split('/')[-1] + ".yaml" + else: + out_filename = query_dict["id"] + ".yaml" + if isinstance(query_dict["pattern"], str): + # example -> message:"java.io.IOException" + message = 'message:"' + query_dict["pattern"] + '"' + elif isinstance(query_dict["pattern"], list): + # example -> + # message: "java.io.IOException" + # AND + # message: "Remote call on" + # AND + # message: "failed" + message = ' AND '.join('message:"' + pattern + '"' for pattern in query_dict["pattern"]) + if 'tags' in query_dict: + if isinstance(query_dict["tags"], str): + message += "AND " + "tags:" + query_dict["tags"] + '"' + elif isinstance(query_dict["tags"], list): + message += " AND (" + ' OR '.join('tags:"' + tags + '"' for tags in query_dict["tags"]) + ")" + if 'suppress-graph' in query_dict: + suppress_graph = bool(query_dict["suppress-graph"]) + er_query = {"query": message, "suppress-graph": suppress_graph} + + with open(os.path.join(elastic_recheck_dest_dir, out_filename), 'w') as out_file: + yaml.dump(er_query, out_file, default_flow_style=False, width=88) diff --git a/src/sova-converter.py b/src/sova-converter.py index c0a0d51..8320058 100644 --- a/src/sova-converter.py +++ b/src/sova-converter.py @@ -3,11 +3,19 @@ import os import yaml import re +""" +This script generates Sova compatible json file from human readable queries. +It takes 'regex' or if 'regex' is not available it converts 'pattern' to regex, from input file and forms +sova compatible json file. + +input: src/data/queries.yml +output: output/sova-pattern-generated.json +""" + dir_path = os.path.dirname(os.path.realpath(__file__)) # Source and destination files queries_src = os.path.join(dir_path, 'data', 'queries.yml') sova_dest = os.path.join(os.path.dirname(dir_path), 'output', 'sova-pattern-generated.json') -# elastic_recheck_dest = os.path.join(os.path.dirname(dir_path), 'output', 'elastic-recheck-pattern-generated.json') with open(queries_src) as in_file: queries_list = yaml.load(in_file, Loader=yaml.BaseLoader) @@ -29,14 +37,11 @@ for query_dict in queries_list['queries']: "name": query_dict["id"], "regex": query_dict["regex"] } - # copy elastic_recheck pattern from query_dict pattern elif 'regex' in query_dict: - # Convert regex to pattern for ER regex_dict = { "name": query_dict["id"], "regex": query_dict["regex"] } - # form elastic_recheck pattern from query_dict regex elif 'pattern' in query_dict: # Convert pattern to regex for Sova if isinstance(query_dict["pattern"], str): @@ -54,7 +59,6 @@ for query_dict in queries_list['queries']: "name": query_dict["id"], "regex": generated_regex } - # copy elastic_recheck pattern from query_dict pattern sova_regex_list.append(regex_dict) patterns = sova_patterns_list.get("console", list()) # if regex_str: diff --git a/tox.ini b/tox.ini index 4002ba2..e3277ba 100644 --- a/tox.ini +++ b/tox.ini @@ -22,6 +22,8 @@ commands = python3 src/sova-converter.py ansible-galaxy collection install -r requirements.yml ansible-playbook playbooks/sova.yml + python3 src/er-converter.py + ansible-playbook playbooks/er.yml passenv = CURL_CA_BUNDLE # https proxies, https://github.com/tox-dev/tox/issues/1437 FORCE_COLOR @@ -36,6 +38,7 @@ passenv = setenv = PIP_DISABLE_PIP_VERSION_CHECK = 1 PRE_COMMIT_COLOR = always + ANSIBLE_NOCOWS = 1 skip_install = true allowlist_externals = bash