Adding script to convert queries to er format

1. The source is src/data/queries.yml. This is the human readable file updated by user.
2. ^ this is converted to files which ER can understand by er_converter.py
3. If there is a bug url in the query entry the corresponding ER file is named <bug_id>.yaml - else the file is named after the query id and shows up as "private bug" in ER dashboard.
4. As a test the playbook er.yml checks if all query files generated can be parsed by ER.
5. Added docstrings for er and sova converters and removed ER specific comments from sova converter.

Depends-On: https://review.opendev.org/c/openstack/tripleo-ci-health-queries/+/780282/

Change-Id: Iea8e6a37b80a95b28ad6e09d041802b7e660d76b

.gitignore

@@ -130,3 +130,6 @@ dmypy.json
 
 # IDE
 .idea/
+
+# tox-report
+report.html

output/README.md

@@ -0,0 +1,7 @@
+# Generated files
+
+All files from within `output/` folder built from other sources. Do not
+attempt to modify them by hand.
+
+You should include the generated files with your CR but never to
+edit them manually.

output/elastic-recheck/1260654.yaml

@@ -0,0 +1,3 @@
+query: message:"java.io.IOException" AND message:"Remote call on" AND message:"failed" AND
+  (tags:"console.html" OR tags:"job-output.txt")
+suppress-graph: true

output/elastic-recheck/curl_re.yaml

@@ -0,0 +1,2 @@
+query: message:"couldn't open file"
+suppress-graph: false

output/elastic-recheck/overcloud_create_failed.yaml

@@ -0,0 +1,2 @@
+query: message:"Stack overcloud CREATE_FAILED"
+suppress-graph: false

playbooks/er.yml

@@ -0,0 +1,35 @@
+---
+- name: Validate that ER can parse what we produce
+  hosts: localhost
+  connection: local
+  gather_facts: true
+
+  vars:
+    er_dir: '../output/elastic-recheck'
+    test_results_dir: '../test_results'
+
+  tasks:
+    - name: Make sure dir for test output exists
+      file:
+        path: "{{ test_results_dir }}"
+        state: directory
+        mode: 0755
+
+    - name: Get list of er files from er directory
+      find:
+        paths: "{{ er_dir }}"
+      register: er_files_output
+
+    - name: Run elastic_recheck_query for all er files
+      vars:
+        output_file: "{{ test_results_dir }}/{{ item.path | basename }}.log"
+      shell: |
+        set -e
+        elastic-recheck-query "{{ item.path }}" > "{{ output_file }}" 2>&1
+        ! grep -q "pyelasticsearch.exceptions.ElasticHttpError" "{{ output_file }}"
+      # This also fails if an exception is found inside the output files
+      loop: "{{ er_files_output.files }}"
+      loop_control:
+        label: "{{ item.path | basename }}"
+      register: er_output
+      changed_when: false

requirements.in

@@ -3,3 +3,4 @@ pydantic>=1.7.4  # MIT
 yq  # Apache
 jsonschema  # MIT
 PyYAML>=5.4.1
+git+https://opendev.org/opendev/elastic-recheck.git#egg=elastic-recheck

requirements.txt

@@ -4,47 +4,294 @@
 #
 #    pip-compile --output-file=requirements.txt requirements.in
 #
+alembic==1.5.8
+    # via
+    #   oslo.db
+    #   subunit2sql
 ansible-base==2.10.8
     # via -r requirements.in
 argcomplete==1.12.2
     # via yq
+argparse==1.4.0
+    # via unittest2
 attrs==20.3.0
     # via jsonschema
+babel==2.9.0
+    # via elastic-recheck
+bcrypt==3.2.0
+    # via paramiko
+certifi==2020.12.5
+    # via requests
 cffi==1.14.5
-    # via cryptography
+    # via
+    #   bcrypt
+    #   cryptography
+    #   pynacl
+chardet==4.0.0
+    # via requests
 cryptography==3.4.7
-    # via ansible-base
+    # via
+    #   ansible-base
+    #   paramiko
+debtcollector==2.2.0
+    # via
+    #   oslo.config
+    #   oslo.db
+    #   oslo.utils
+decorator==5.0.7
+    # via sqlalchemy-migrate
+distro==1.5.0
+    # via lazr.restfulclient
+docutils==0.17.1
+    # via python-daemon
+git+https://opendev.org/opendev/elastic-recheck.git#egg=elastic-recheck
+    # via -r requirements.in
+extras==1.0.0
+    # via
+    #   python-subunit
+    #   testtools
+fixtures==3.0.0
+    # via testtools
+gerritlib==0.10.0
+    # via elastic-recheck
+greenlet==1.0.0
+    # via sqlalchemy
+httplib2==0.19.1
+    # via
+    #   elastic-recheck
+    #   launchpadlib
+    #   lazr.restfulclient
+idna==2.10
+    # via requests
+importlib-metadata==4.0.1
+    # via keyring
+irc==19.0.1
+    # via elastic-recheck
+iso8601==0.1.14
+    # via oslo.utils
+jaraco.classes==3.2.1
+    # via jaraco.collections
+jaraco.collections==3.3.0
+    # via irc
+jaraco.functools==3.3.0
+    # via
+    #   irc
+    #   jaraco.text
+    #   tempora
+jaraco.logging==3.1.0
+    # via irc
+jaraco.stream==3.0.2
+    # via irc
+jaraco.text==3.5.0
+    # via
+    #   irc
+    #   jaraco.collections
 jinja2==2.11.3
-    # via ansible-base
+    # via
+    #   ansible-base
+    #   elastic-recheck
 jsonschema==3.2.0
     # via -r requirements.in
+keyring==23.0.1
+    # via launchpadlib
+launchpadlib==1.10.13
+    # via elastic-recheck
+lazr.restfulclient==0.14.3
+    # via
+    #   elastic-recheck
+    #   launchpadlib
+lazr.uri==1.0.5
+    # via
+    #   launchpadlib
+    #   wadllib
+linecache2==1.0.0
+    # via traceback2
+lockfile==0.12.2
+    # via
+    #   elastic-recheck
+    #   python-daemon
+mako==1.1.4
+    # via alembic
 markupsafe==1.1.1
-    # via jinja2
+    # via
+    #   jinja2
+    #   mako
+more-itertools==8.7.0
+    # via
+    #   irc
+    #   jaraco.classes
+    #   jaraco.functools
+netaddr==0.8.0
+    # via
+    #   oslo.config
+    #   oslo.utils
+netifaces==0.10.9
+    # via oslo.utils
+oauthlib==3.1.0
+    # via lazr.restfulclient
+oslo.config==8.5.0
+    # via
+    #   oslo.db
+    #   subunit2sql
+oslo.db==8.5.0
+    # via subunit2sql
+oslo.i18n==5.0.1
+    # via
+    #   oslo.config
+    #   oslo.db
+    #   oslo.utils
+oslo.utils==4.8.0
+    # via oslo.db
 packaging==20.9
-    # via ansible-base
+    # via
+    #   ansible-base
+    #   oslo.utils
+paramiko==2.7.2
+    # via gerritlib
+pbr==5.5.1
+    # via
+    #   debtcollector
+    #   elastic-recheck
+    #   fixtures
+    #   gerritlib
+    #   oslo.db
+    #   oslo.i18n
+    #   oslo.utils
+    #   sqlalchemy-migrate
+    #   stevedore
+    #   subunit2sql
+    #   testresources
+    #   testscenarios
+    #   testtools
 pycparser==2.20
     # via cffi
 pydantic==1.8.1
     # via -r requirements.in
+pyelasticsearch==0.7.1
+    # via elastic-recheck
+pymysql==1.0.2
+    # via elastic-recheck
+pynacl==1.4.0
+    # via paramiko
 pyparsing==2.4.7
-    # via packaging
+    # via
+    #   httplib2
+    #   oslo.utils
+    #   packaging
 pyrsistent==0.17.3
     # via jsonschema
+python-daemon==2.3.0
+    # via elastic-recheck
+python-dateutil==2.8.1
+    # via
+    #   alembic
+    #   elastic-recheck
+    #   subunit2sql
+python-editor==1.0.4
+    # via alembic
+python-mimeparse==1.6.0
+    # via testtools
+python-subunit==1.4.0
+    # via subunit2sql
+pytz==2021.1
+    # via
+    #   babel
+    #   elastic-recheck
+    #   irc
+    #   oslo.utils
+    #   tempora
 pyyaml==5.4.1
     # via
     #   -r requirements.in
     #   ansible-base
+    #   elastic-recheck
+    #   oslo.config
     #   yq
+requests==2.25.1
+    # via
+    #   elastic-recheck
+    #   oslo.config
+    #   pyelasticsearch
+rfc3986==1.4.0
+    # via oslo.config
+simplejson==3.17.2
+    # via pyelasticsearch
 six==1.15.0
-    # via jsonschema
+    # via
+    #   bcrypt
+    #   debtcollector
+    #   fixtures
+    #   gerritlib
+    #   jsonschema
+    #   launchpadlib
+    #   lazr.restfulclient
+    #   oslo.i18n
+    #   pyelasticsearch
+    #   pynacl
+    #   python-dateutil
+    #   sqlalchemy-migrate
+    #   subunit2sql
+    #   testtools
+    #   unittest2
+sqlalchemy-migrate==0.13.0
+    # via oslo.db
+sqlalchemy==1.4.11
+    # via
+    #   alembic
+    #   elastic-recheck
+    #   oslo.db
+    #   sqlalchemy-migrate
+    #   subunit2sql
+sqlparse==0.4.1
+    # via sqlalchemy-migrate
+stevedore==3.3.0
+    # via
+    #   oslo.config
+    #   oslo.db
+    #   subunit2sql
+subunit2sql==1.10.0
+    # via elastic-recheck
+tempita==0.5.2
+    # via sqlalchemy-migrate
+tempora==4.0.2
+    # via
+    #   irc
+    #   jaraco.logging
+testresources==2.0.1
+    # via
+    #   launchpadlib
+    #   oslo.db
+testscenarios==0.5.0
+    # via oslo.db
+testtools==2.4.0
+    # via
+    #   fixtures
+    #   python-subunit
+    #   testscenarios
 toml==0.10.2
     # via yq
+traceback2==1.4.0
+    # via
+    #   testtools
+    #   unittest2
 typing-extensions==3.7.4.3
     # via pydantic
+unittest2==1.1.0
+    # via testtools
+urllib3==1.26.4
+    # via requests
+wadllib==1.3.5
+    # via
+    #   launchpadlib
+    #   lazr.restfulclient
+wrapt==1.12.1
+    # via debtcollector
 xmltodict==0.12.0
     # via yq
 yq==2.12.0
     # via -r requirements.in
+zipp==3.4.1
+    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

src/er-converter.py

@@ -0,0 +1,58 @@
+import json
+import os
+import yaml
+import re
+
+"""
+This script generates Elastic-Recheck compatible yaml files from human readable queries.
+It takes 'pattern' and 'tags', both of which can be strings or lists, from input file and forms
+an elastic recheck query. This query is written in a file with filename same as the bug number if bug URL is provided
+or else the id from input file.
+
+input: src/data/queries.yml
+output: output/elastic-recheck/<id/bug_no>.yaml
+"""
+
+dir_path = os.path.dirname(os.path.realpath(__file__))
+# Source and destination files
+queries_src = os.path.join(dir_path, 'data', 'queries.yml')
+elastic_recheck_dest_dir = os.path.join(os.path.dirname(dir_path), 'output', 'elastic-recheck')
+# Make sure dest dir for er is present
+if not os.path.exists(elastic_recheck_dest_dir):
+    os.makedirs(elastic_recheck_dest_dir)
+
+with open(queries_src) as in_file:
+    queries_list = yaml.load(in_file, Loader=yaml.BaseLoader)
+
+for query_dict in queries_list['queries']:
+    if "pattern" not in query_dict:
+        continue
+    # Assuming "pattern" is always present if it needs to be shown in ER
+    suppress_graph = False  # default
+    message = ''
+    if "url" in query_dict:
+        out_filename = query_dict["url"].split('/')[-1] + ".yaml"
+    else:
+        out_filename = query_dict["id"] + ".yaml"
+    if isinstance(query_dict["pattern"], str):
+        # example -> message:"java.io.IOException"
+        message = 'message:"' + query_dict["pattern"] + '"'
+    elif isinstance(query_dict["pattern"], list):
+        # example ->
+        # message: "java.io.IOException"
+        # AND
+        # message: "Remote call on"
+        # AND
+        # message: "failed"
+        message = ' AND '.join('message:"' + pattern + '"' for pattern in query_dict["pattern"])
+    if 'tags' in query_dict:
+        if isinstance(query_dict["tags"], str):
+            message += "AND " + "tags:" + query_dict["tags"] + '"'
+        elif isinstance(query_dict["tags"], list):
+            message += " AND (" + ' OR '.join('tags:"' + tags + '"' for tags in query_dict["tags"]) + ")"
+    if 'suppress-graph' in query_dict:
+        suppress_graph = bool(query_dict["suppress-graph"])
+    er_query = {"query": message, "suppress-graph": suppress_graph}
+
+    with open(os.path.join(elastic_recheck_dest_dir, out_filename), 'w') as out_file:
+        yaml.dump(er_query, out_file, default_flow_style=False, width=88)

src/sova-converter.py

@@ -3,11 +3,19 @@ import os
 import yaml
 import re
 
+"""
+This script generates Sova compatible json file from human readable queries.
+It takes 'regex' or if 'regex' is not available it converts 'pattern' to regex, from input file and forms
+sova compatible json file.
+
+input: src/data/queries.yml
+output: output/sova-pattern-generated.json
+"""
+
 dir_path = os.path.dirname(os.path.realpath(__file__))
 # Source and destination files
 queries_src = os.path.join(dir_path, 'data', 'queries.yml')
 sova_dest = os.path.join(os.path.dirname(dir_path), 'output', 'sova-pattern-generated.json')
-# elastic_recheck_dest = os.path.join(os.path.dirname(dir_path), 'output', 'elastic-recheck-pattern-generated.json')
 
 with open(queries_src) as in_file:
     queries_list = yaml.load(in_file, Loader=yaml.BaseLoader)
@@ -29,14 +37,11 @@ for query_dict in queries_list['queries']:
             "name": query_dict["id"],
             "regex": query_dict["regex"]
         }
-        # copy elastic_recheck pattern from query_dict pattern
     elif 'regex' in query_dict:
-        # Convert regex to pattern for ER
         regex_dict = {
             "name": query_dict["id"],
             "regex": query_dict["regex"]
         }
-        # form elastic_recheck pattern from query_dict regex
     elif 'pattern' in query_dict:
         # Convert pattern to regex for Sova
         if isinstance(query_dict["pattern"], str):
@@ -54,7 +59,6 @@ for query_dict in queries_list['queries']:
                 "name": query_dict["id"],
                 "regex": generated_regex
             }
-        # copy elastic_recheck pattern from query_dict pattern
     sova_regex_list.append(regex_dict)
     patterns = sova_patterns_list.get("console", list())
     # if regex_str:

tox.ini

@@ -22,6 +22,8 @@ commands =
   python3 src/sova-converter.py
   ansible-galaxy collection install -r requirements.yml
   ansible-playbook playbooks/sova.yml
+  python3 src/er-converter.py
+  ansible-playbook playbooks/er.yml
 passenv =
   CURL_CA_BUNDLE  # https proxies, https://github.com/tox-dev/tox/issues/1437
   FORCE_COLOR
@@ -36,6 +38,7 @@ passenv =
 setenv =
   PIP_DISABLE_PIP_VERSION_CHECK = 1
   PRE_COMMIT_COLOR = always
+  ANSIBLE_NOCOWS = 1
 skip_install = true
 allowlist_externals =
   bash