diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 531c1a5..eb9c178 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: # https://github.com/pre-commit/mirrors-isort/issues/9#issuecomment-624404082 - --filter-files - repo: https://github.com/python/black.git - rev: 20.8b1 + rev: 21.4b2 hooks: - id: black language_version: python3 diff --git a/README.md b/README.md index 80dba24..6ce2196 100644 --- a/README.md +++ b/README.md @@ -77,3 +77,43 @@ just use it. To avoid using a particular query on a particular backend we can make use of ``skip: ['er', 'artcl']``. + + +## Parameters in a query + +| Key | Required | Type | Sova/ER | Description | +| ------------- |:--------:| ------------------------:|-------------:|------------:| +| id | Required | string, unique | Sova | This id will appear in sova.log when this bug is encountered. | +| pattern | Optional | string / list of strings | ER (,Sova) | This used as it is by Elastic recheck to form logstash queries like ``` message:foo ``` or ``` message:foo AND message:bar ``` If not present Elastic Recheck skips this entry | +| regex | Optional | string / list of strings | Sova | Sova searches for this regex in the log files. If not present the pattern string is converted to regex (escaping special characters) and used by sova. | +| tags | Optional | string / list of strings | ER | Elastic recheck uses this to form logstash queries like ``` AND (tags:"console.html" OR tags:"job-output.txt")``` | +| url | Optional | string | ER | Launchpad bug URL. If present the bug number and details are displayed on ER dashboard. If not the graph is titled Unknow/private bug | +| suppress-graph| Optional | bool | ER | Decides whether or not to display a graph for this bug in Elastic recheck dashboard | + +## Adding a new item in queries.yml + +To add a new query (optional: from a bug) + +- Add a query item in [src/data/queries.yml](https://opendev.org/openstack/tripleo-ci-health-queries/src/branch/master/src/data/queries.yml) + - id : unique ID + - pattern: The error message(s) to look for. + - regex: If it is a complicated error message you can add a regex for sova to use. + - tags: tag(s) for logstash query + - url: Launchpad Bug URL + - suppress-graph: To display/suppress graph for the bug in ER + +To move an existing sova regex from sova-patterns.json to queries.yml + +- Choose an item from ``` regexes ``` in output/sova-patterns.json. This is the file that sova currently uses. + (Do not look at patterns for now) +- If the regex feels still relevant, vote for it here https://docs.google.com/spreadsheets/d/16rqgaSSoQrYNjsI4q0YInJxrOOJL3xyP3t3D6ZpluFY/edit#gid=846893892 +- Add a query entry in src/data/queries.yml + - id: `name` of the regexes item you chose. + - pattern: If you want this error to be shown in ER dashboard add a string here corresponding to the regex + - regex: `regex` of the regexes item you chose. + +- Add a sample string matching the pattern (or regex if you have a special regex) in samples/errors-testing.err. +As the part of check job a task calls sova to check this file and makes sure there is a match for all the regexes + in queries.yml. +- Run tox. If it passes it should create files in output/elastic-recheck and update sova-pattern-generated. +- Commit all these files. diff --git a/src/data/queries-example.yml b/src/data/queries-example.yml index bdf01a4..1479573 100644 --- a/src/data/queries-example.yml +++ b/src/data/queries-example.yml @@ -1,11 +1,14 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/rdo-infra/queries/main/output/queries-schema.json queries: - id: pip-no-matching-distribution # sort key - name: "PIP failed to find package" # optional (aka 'reason'), id is used instead - pattern: "ERROR: No matching distribution found for" # required, can be list! + # optionals: + pattern: "ERROR: No matching distribution found for" # required, can be list! This string is used by + # Elastic Recheck to form logstash query message:"" + # Sova in collect-logs + name: "PIP failed to find package" # optional (aka 'reason'), id is used instead files: [] # list of glob patterns, narrows down searching - regex: false # optional, default is false + regex: string # optional, default is false url: ... # str or list[str], issue links to lp, bz,... category: ... # see Categories tags: [] # see Tags