From 146d3d7fff1f3284f7431bc1c288027185a4ed8f Mon Sep 17 00:00:00 2001 From: frenzyfriday Date: Mon, 15 Mar 2021 11:42:40 +0100 Subject: [PATCH] Adding script to convert queries to sova format - add a few sova regexes in queries - updated requirements file to include PyYaml used in the converter script - Added a converter script which : - Parses queries.yml - If an entry has regex it forms sova query with regex, if the entry has patterns but no regex it converts the pattern (string) to regex (escapes characters) and forms sova query - Writes the sova output to sova-pattern-generated.json (so as not to overwrite existing sova_pattern.json right now) Change-Id: Iae3df0d901f2b25bdb27f1b3bea5042f82128dcf --- ansible.cfg | 4 ++ bindep.txt | 3 ++ hosts | 1 + output/queries-schema.json | 16 +++++-- output/sova-pattern-generated.json | 56 +++++++++++++++++++++++ playbooks/sova.yml | 35 +++++++++++++++ requirements.in | 3 +- requirements.txt | 17 +++++++ requirements.yml | 4 ++ samples/errors-testing.err | 3 ++ src/data/queries.yml | 6 ++- src/model.py | 4 +- src/sova-converter.py | 72 ++++++++++++++++++++++++++++++ tox.ini | 5 +++ 14 files changed, 220 insertions(+), 9 deletions(-) create mode 100644 ansible.cfg create mode 100644 hosts create mode 100644 output/sova-pattern-generated.json create mode 100644 playbooks/sova.yml create mode 100644 requirements.yml create mode 100644 samples/errors-testing.err create mode 100644 src/sova-converter.py diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 0000000..54f87f0 --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,4 @@ +[defaults] +interpreter_python=auto_silent +inventory = hosts +nocows = 0 diff --git a/bindep.txt b/bindep.txt index ca78288..08c0182 100644 --- a/bindep.txt +++ b/bindep.txt @@ -1 +1,4 @@ jq +# requried by sova ansible module (not inside the venv) +python3-yaml [platform:ubuntu] +python3-pyyaml [platform:rpm !platform:rhel-7 !platform:centos-7] diff --git a/hosts b/hosts new file mode 100644 index 0000000..ff5df5d --- /dev/null +++ b/hosts @@ -0,0 +1 @@ +localhost connection=local diff --git a/output/queries-schema.json b/output/queries-schema.json index 9fea83c..e459a70 100644 --- a/output/queries-schema.json +++ b/output/queries-schema.json @@ -85,8 +85,17 @@ }, "regex": { "title": "Regex", - "default": false, - "type": "boolean" + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } + ] }, "multiline": { "title": "Multiline", @@ -103,8 +112,7 @@ } }, "required": [ - "id", - "pattern" + "id" ], "additionalProperties": false } diff --git a/output/sova-pattern-generated.json b/output/sova-pattern-generated.json new file mode 100644 index 0000000..c37aff8 --- /dev/null +++ b/output/sova-pattern-generated.json @@ -0,0 +1,56 @@ +{ + "patterns": { + "console": [ + { + "id": "java_io_exception_remote_call", + "logstash": "", + "msg": "java_io_exception_remote_call", + "pattern": "java_io_exception_remote_call", + "tag": "info" + }, + { + "id": "overcloud_create_failed", + "logstash": "", + "msg": "overcloud_create_failed", + "pattern": "overcloud_create_failed", + "tag": "info" + }, + { + "id": "timeout_re", + "logstash": "", + "msg": "timeout_re", + "pattern": "timeout_re", + "tag": "info" + }, + { + "id": "curl_re", + "logstash": "", + "msg": "curl_re", + "pattern": "curl_re", + "tag": "info" + } + ] + }, + "regexes": [ + { + "name": "java_io_exception_remote_call", + "regex": [ + "java\\.io\\.IOException", + "Remote\\ call\\ on", + "failed" + ] + }, + { + "name": "overcloud_create_failed", + "regex": "Stack\\ overcloud\\ CREATE_FAILED" + }, + { + "name": "timeout_re", + "regex": "Killed\\s+timeout -s 9" + }, + { + "name": "curl_re", + "regex": "curl. .*? couldn\\\\'t open file \"(.*?)\"" + } + ] +} diff --git a/playbooks/sova.yml b/playbooks/sova.yml new file mode 100644 index 0000000..a6fdb69 --- /dev/null +++ b/playbooks/sova.yml @@ -0,0 +1,35 @@ +--- +- name: Validate that sova can parse what we produce + hosts: localhost + connection: local + gather_facts: true + collections: + - tripleo.collect_logs + vars: + sova_cfg_file: "{{ lookup('file', playbook_dir + '/../output/sova-pattern-generated.json') }}" + samples_dir: "{{ (playbook_dir, '../samples/') | path_join | realpath }}" + samples_log: "{{ samples_dir }}/errors-testing.err" + tasks: + - name: Display file to be tested + debug: + var: sova_cfg_file + + - name: Run sova task + sova: + config: "{{ sova_cfg_file }}" + files: + console: "{{ samples_log }}" + result: "sova.log" + result_file_dir: "{{ (playbook_dir, '../output') | path_join | realpath }}" + register: result + + - name: Display sova result + debug: + var: result + + - name: Fail it unexpected result is detected + fail: + msg: "Unexpected result: {{ result }}" + when: > + samples_log not in result.processed_files + or result.file_written != 'sova.log' diff --git a/requirements.in b/requirements.in index a4e7a70..30ce364 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,5 @@ -pydantic>=1.7.4 # MIT +ansible-base>=2.10 # GPL +pydantic>=1.7.4 # MIT yq # Apache jsonschema # MIT PyYAML>=5.4.1 diff --git a/requirements.txt b/requirements.txt index 54dca00..25a450b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,19 +4,36 @@ # # pip-compile --output-file=requirements.txt requirements.in # +ansible-base==2.10.8 + # via -r requirements.in argcomplete==1.12.2 # via yq attrs==20.3.0 # via jsonschema +cffi==1.14.5 + # via cryptography +cryptography==3.4.7 + # via ansible-base +jinja2==2.11.3 + # via ansible-base jsonschema==3.2.0 # via -r requirements.in +markupsafe==1.1.1 + # via jinja2 +packaging==20.9 + # via ansible-base +pycparser==2.20 + # via cffi pydantic==1.8.1 # via -r requirements.in +pyparsing==2.4.7 + # via packaging pyrsistent==0.17.3 # via jsonschema pyyaml==5.4.1 # via # -r requirements.in + # ansible-base # yq six==1.15.0 # via jsonschema diff --git a/requirements.yml b/requirements.yml new file mode 100644 index 0000000..cbbc39a --- /dev/null +++ b/requirements.yml @@ -0,0 +1,4 @@ +collections: + - name: git+https://opendev.org/openstack/ansible-role-collect-logs + type: git + version: master diff --git a/samples/errors-testing.err b/samples/errors-testing.err new file mode 100644 index 0000000..2a3de5c --- /dev/null +++ b/samples/errors-testing.err @@ -0,0 +1,3 @@ +couldn't open file +Stack overcloud CREATE_FAILED +AnsibleUndefinedVariable diff --git a/src/data/queries.yml b/src/data/queries.yml index ef65e86..8f6ecf6 100644 --- a/src/data/queries.yml +++ b/src/data/queries.yml @@ -16,5 +16,7 @@ queries: pattern: "Stack overcloud CREATE_FAILED" # from https://opendev.org/opendev/elastic-recheck/src/branch/master/queries/1260654.yaml - id: timeout_re - pattern: Killed\s+timeout -s 9 - regex: true + regex: 'Killed\s+timeout -s 9' + - id: curl_re + pattern: "couldn't open file" + regex: 'curl. .*? couldn\\''t open file "(.*?)"' diff --git a/src/model.py b/src/model.py index b0095bb..24cd210 100644 --- a/src/model.py +++ b/src/model.py @@ -7,7 +7,7 @@ from pydantic import BaseModel, Field, Extra, HttpUrl class Query(BaseModel): id: str name: Optional[str] - pattern: Union[List[str], str] + pattern: Optional[Union[List[str], str]] category: Optional[str] url: Optional[Union[List[HttpUrl], HttpUrl]] @@ -19,7 +19,7 @@ class Query(BaseModel): description="Used for elastic-recheck") # artcl/sove specific fields - regex: Optional[bool] = False + regex: Optional[Union[List[str], str]] # https://opendev.org/openstack/ansible-role-collect-logs/src/branch/master/vars/sova-patterns.yml#L47 multiline: Optional[bool] = False files: Optional[List[str]] = Field( diff --git a/src/sova-converter.py b/src/sova-converter.py new file mode 100644 index 0000000..c0a0d51 --- /dev/null +++ b/src/sova-converter.py @@ -0,0 +1,72 @@ +import json +import os +import yaml +import re + +dir_path = os.path.dirname(os.path.realpath(__file__)) +# Source and destination files +queries_src = os.path.join(dir_path, 'data', 'queries.yml') +sova_dest = os.path.join(os.path.dirname(dir_path), 'output', 'sova-pattern-generated.json') +# elastic_recheck_dest = os.path.join(os.path.dirname(dir_path), 'output', 'elastic-recheck-pattern-generated.json') + +with open(queries_src) as in_file: + queries_list = yaml.load(in_file, Loader=yaml.BaseLoader) + +sova_regex_list = [] +sova_patterns_list = {"console": []} +sova_dict = { + 'regexes': sova_regex_list, + 'patterns': sova_patterns_list +} + +for query_dict in queries_list['queries']: + regex_dict = {} + regex_str = query_dict.get("regex", "") + + if {'regex', 'pattern'} <= query_dict.keys(): + # No pattern/regex conversion + regex_dict = { + "name": query_dict["id"], + "regex": query_dict["regex"] + } + # copy elastic_recheck pattern from query_dict pattern + elif 'regex' in query_dict: + # Convert regex to pattern for ER + regex_dict = { + "name": query_dict["id"], + "regex": query_dict["regex"] + } + # form elastic_recheck pattern from query_dict regex + elif 'pattern' in query_dict: + # Convert pattern to regex for Sova + if isinstance(query_dict["pattern"], str): + generated_regex = re.escape(query_dict["pattern"]) + regex_dict = { + "name": query_dict["id"], + "regex": generated_regex + } + regex_str = generated_regex + elif isinstance(query_dict["pattern"], list): + generated_regex = [] + for item in query_dict["pattern"]: + generated_regex.append(re.escape(item)) + regex_dict = { + "name": query_dict["id"], + "regex": generated_regex + } + # copy elastic_recheck pattern from query_dict pattern + sova_regex_list.append(regex_dict) + patterns = sova_patterns_list.get("console", list()) + # if regex_str: + patterns.append({ + "id": query_dict['id'], + "logstash": "", + "msg": query_dict['id'], + "pattern": regex_dict['name'], + "tag": "info" + }) + +with open(sova_dest, 'w') as out_file: + json.dump(sova_dict, out_file, indent=4, sort_keys=True) + # Adds newline to pass lint + out_file.write("\n") diff --git a/tox.ini b/tox.ini index c770342..4002ba2 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,9 @@ deps = commands = python3 src/model.py bash -c "cat src/data/queries.yml | yq | jsonschema -i /dev/stdin output/queries-schema.json" + python3 src/sova-converter.py + ansible-galaxy collection install -r requirements.yml + ansible-playbook playbooks/sova.yml passenv = CURL_CA_BUNDLE # https proxies, https://github.com/tox-dev/tox/issues/1437 FORCE_COLOR @@ -27,6 +30,7 @@ passenv = PYTEST_* # allows developer to define their own preferences PY_COLORS REQUESTS_CA_BUNDLE # https proxies + TERM SSL_CERT_FILE # https proxies # recreate = True setenv = @@ -35,6 +39,7 @@ setenv = skip_install = true allowlist_externals = bash + cat [testenv:deps] description = Update dependency lock files