Run Sova patterns parsing in CI jobs

Using regexps and string patterns, analyze logs and save failure
messages in file.
The original service code is in https://github.com/sshnaidm/sova

Change-Id: Ib55d1b025c4ad9e1775d7918d052ef5a27682c3d
This commit is contained in:
Sagi Shnaidman 2019-08-26 21:39:04 +03:00 committed by wes hayutin
parent a1025122b7
commit 50e357a4a3
11 changed files with 1108 additions and 2 deletions

View File

@ -126,8 +126,38 @@ Publishing related
- ``artcl_collect_sosreport`` true/false If true, create and
collect a sosreport for each host.
Example Playbook
----------------
Logs parsing
~~~~~~~~~~~~
"Sova" module parses logs for known patterns and returns messages that were
found. Patterns are tagged by issues types, like "infra", "code", etc.
Patterns are located in file sova-patterns.yml in vars/ directory.
- ``config`` - patterns loaded from file
- ``files`` - files and patterns sections match
- ``result`` - path to file to write a result of parsing
- ``result_file_dir`` - directory to write a file with patterns in name
Example of usage of "sova" module:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. code:: yaml
---
- name: Run sova task
sova:
config: "{{ pattern_config }}"
files:
console: "{{ ansible_user_dir }}/workspace/logs/quickstart_install.log"
errors: "/var/log/errors.txt"
"ironic-conductor": "/var/log/containers/ironic/ironic-conductor.log"
syslog: "/var/log/journal.txt"
logstash: "/var/log/extra/logstash.txt"
result: "{{ ansible_user_dir }}/workspace/logs/failures_file"
result_file_dir: "{{ ansible_user_dir }}/workspace/logs"
Example Role Playbook
---------------------
.. code:: yaml
@ -160,6 +190,7 @@ has several simple rules:
5. All other lines, including shell comments, will be indented by four
spaces.
Enabling sosreport Collection
-----------------------------

192
library/sova.py Normal file
View File

@ -0,0 +1,192 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, division, print_function
__metaclass__ = type
import os
from copy import deepcopy
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.sova_lib import Pattern, parse
ANSIBLE_METADATA = {
'metadata_version': '0.1',
'status': ['preview'],
'supported_by': 'community'
}
DOCUMENTATION = """
module: sova
author:
- "Sagi Shnaidman (@sshnaidm)"
version_added: '2.7'
short_description: Parse CI jobs files for known failures
notes: []
description:
- Parse CI job files and find there known patterns of failures
requirements:
- "Better to use with 'regex' module installed"
options:
files:
description:
- Dictionary of patterns file name and file location.
Patterns are divided by sections in config file, match each section
to the file path on the host, It will search these patterns from this
section in the given file.
required: True
type: dict
result:
description:
- Path to file where to write result message.
type: path
result_file_dir:
description:
- Directory where to create a file with result message and name
of file. For example for pattern 'Overcloud failed on host' will be
created file Overcloud_failed_on_host.log in this directory.
It helps to know what is the reason without opening actually the file.
type: path
"""
EXAMPLES = """
- name: Run sova task
sova:
files:
console: /var/log/job-output.txt.gz
errors: /var/log/errors.txt.txt.gz
"ironic-conductor": /var/log/ironic-conductor.log.txt.gz
syslog: /var/log/journal.txt.gz
logstash: /var/log/logstash.txt.gz
bmc: /var/log/bmc-console.log
result: /home/zuul/result_file
result_file_dir: /home/zuul/workspace/logs/
"""
RETURN = """
processed_files:
description:
- Files which have been processed by module
returned: if changed
type: list
sample: [
"/tmp/var/log/job-output.txt.gz",
"/tmp/var/log/errors.txt.txt.gz",
"/tmp/var/log/ironic-conductor.log.txt.gz"
]
message:
description:
- Text with all messages about failures
returned: if changed
type: list
sample: 'Overcloud stack: FAILED.'
tags:
description:
- Tags of patterns which were found in files
returned: if changed
type: list
sample: ["info"]
file_name_written:
description:
- Path of file which written with message as filename
returned: if changed
type: str
sample: '/var/log/_Overcloud_stack__FAILED.log'
file_written:
description:
- Path of file where written result message and reason.
returned: if changed
type: str
sample: '/var/log/result_file'
"""
def format_msg_filename(text):
for s in (" ", ":", ".", "/", ",", "'", ):
text = text.replace(s, "_")
return "_" + text.rstrip("_") + ".log"
def main():
module = AnsibleModule(
argument_spec=dict(
config=dict(type='dict', default={}),
files=dict(type='dict', default={}),
result=dict(type='path'),
result_file_dir=dict(type='path'),
))
if not module.params['files']:
module.fail_json(msg="Files for logs parsing have to be provided!")
existing_files = []
for pattern_file in module.params['files']:
file_ = module.params['files'][pattern_file]
if os.path.exists(file_):
existing_files.append(file_)
if not existing_files:
results = {"processed_files": [], 'changed': False}
module.exit_json(**results)
dict_patterns = deepcopy(module.params['config'])
pattern = Pattern(dict_patterns)
PATTERNS = pattern.patterns
for name in module.params['files']:
if name not in PATTERNS:
module.fail_json(msg="File name %s wasn't found in [%s]" % (
name, ", ".join(list(PATTERNS.keys()))))
messages, tags = [], []
for name, file_ in module.params['files'].items():
if module.params['files'][name] not in existing_files:
continue
ids, msgs = parse(file_, PATTERNS[name])
found = [i for i in PATTERNS[name] if i['id'] in ids]
msg_tags = [i['tag'] for i in found if i.get('tag')]
messages += msgs
tags += msg_tags
messages = list(set(messages))
tags = list(set(tags))
if 'infra' in tags:
reason = 'infra'
elif 'code' in tags:
reason = 'code'
else:
reason = 'unknown'
text = " ".join(messages) or "No failure reason found"
file_name = format_msg_filename(text)
result = {'changed': True, "processed_files": existing_files}
result.update({'message': text})
result.update({'tags': tags})
if module.params['result'] and messages:
try:
with open(module.params['result'], "w") as f:
f.write(text + "\n")
f.write("Reason: " + reason + "\n")
result.update({'file_written': module.params['result']})
except Exception as e:
module.fail_json(
msg="Can't write result to file %s: %s" % (
module.params['result'], str(e)))
if module.params['result_file_dir']:
log_file = os.path.join(module.params['result_file_dir'], file_name)
try:
with open(log_file, "w") as f:
f.write(text + "\n")
f.write("Reason: " + reason + "\n")
result.update({'file_name_written': log_file})
except Exception as e:
module.fail_json(
msg="Can't write result to file %s: %s" % (log_file, str(e)))
module.exit_json(**result)
if __name__ == '__main__':
main()

102
module_utils/sova_lib.py Normal file
View File

@ -0,0 +1,102 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import gzip
import logging
import yaml
try:
import regex as regex_module
except ImportError:
import re as regex_module
logging.basicConfig(
format=('%(asctime)s - %(name)s - %(levelname)s - '
'%(module)s.%(funcName)s:%(lineno)d - %(message)s'))
log = logging.getLogger('parser')
log.setLevel(logging.ERROR)
class Pattern(object):
def __init__(self, data):
self.data = data
self.load_yaml()
self.setup_regexes()
self.setup_patterns()
def load_yaml(self):
if isinstance(self.data, dict):
self.config = self.data
else:
self.config = yaml.safe_load(self.data)
def setup_regexes(self):
self.regexes = {}
if self.config:
for regexp in self.config.get('regexes', []):
flags = []
if regexp.get('multiline'):
flags.append(regex_module.MULTILINE)
self.regexes[regexp.get('name')] = regex_module.compile(
r'{}'.format(regexp.get('regex')), *flags)
def setup_patterns(self):
self._patterns = self.config.get('patterns', {})
if self._patterns:
for key in self._patterns:
for p in self._patterns[key]:
if p['pattern'] in self.regexes:
p['pattern'] = self.regexes[p['pattern']]
if p['logstash'] in self.regexes:
p['logstash'] = self.regexes[p['logstash']]
@property
def patterns(self):
return self._patterns
def line_match(pat, line, exclude=None):
if isinstance(pat, str):
return pat in line
found = pat.search(line)
if not found:
return False
if found.groups():
if exclude:
if any([i in found.group(1) for i in exclude]):
return False
return found.group(1)
return True
def parse(text_file, patterns):
ids = []
msgs = []
if text_file.split(".")[-1] == "gz":
open_func = gzip.open
else:
open_func = open
with open_func(text_file, "rt") as finput:
text = finput.read()
for p in patterns:
line_matched = line_match(
p["pattern"], text, exclude=p.get("exclude"))
if line_matched:
log.debug("Found pattern {} in file {}".format(
repr(p), text_file))
ids.append(p["id"])
msgs.append(p["msg"].format(line_matched))
return list(set(ids)), list(set(msgs))

View File

@ -0,0 +1,28 @@
---
driver:
name: delegated
options:
managed: false
ansible_connection_options:
ansible_connection: local
log: true
lint:
name: yamllint
platforms:
- name: instance
provisioner:
name: ansible
lint:
name: ansible-lint
# Need to fix a whole role
options:
x:
- '301'
- '302'
- '303'
- '305'
verifier:
name: ansible
lint:
name: ansible-lint

View File

@ -0,0 +1,10 @@
---
- name: Converge
hosts: all
tasks:
- name: "Include ansible-role-collect-logs"
include_role:
name: "ansible-role-collect-logs"
tasks_from: sova.yml
tags:
- molecule-idempotence-notest

20
molecule/sova/prepare.yml Normal file
View File

@ -0,0 +1,20 @@
---
- name: Prepare
hosts: all
tasks:
- name: Install python YAML module
pip:
name: pyyaml
extra_args: --user
- name: Prepare the console file directory
file:
path: '{{ ansible_user_dir }}/workspace/logs/'
state: directory
- name: Create a sample console file
copy:
content: |
No valid host was found. There are not enough hosts
dest: '{{ ansible_user_dir }}/workspace/logs/quickstart_install.log'

24
molecule/sova/verify.yml Normal file
View File

@ -0,0 +1,24 @@
---
- hosts: all
tasks:
- name: Ensure all files exists
stat:
path: "{{ ansible_user_dir }}/workspace/logs/failures_file"
register: failures_file
loop:
- "{{ ansible_user_dir }}/workspace/logs/failures_file"
- "{{ ansible_user_dir }}/workspace/logs/_No_valid_host_was_found.log"
- name: Ensure all files exists - test
assert:
that:
- item.stat.exists
loop: "{{ failures_file.results }}"
- name: Check if we have strings in failures_file
command: grep 'No valid host was found' {{ ansible_user_dir }}/workspace/logs/failures_file
changed_when: false
- name: Check if we have strings in error file
command: grep 'No valid host was found' {{ ansible_user_dir }}/workspace/logs/_No_valid_host_was_found.log
changed_when: false

View File

@ -26,6 +26,8 @@ data_files =
usr/local/share/ansible/roles/collect-logs/templates = templates/*
usr/local/share/ansible/roles/collect-logs/files = files/*
usr/local/share/ansible/roles/collect-logs/library = library/*
usr/local/share/ansible/roles/collect-logs/module_utils = module_utils/*
usr/local/share/ansible/roles/collect-logs/vars = vars/*
usr/local/share/ansible/roles/collect-logs/scripts = scripts/*
usr/local/share/ansible/roles/collect-logs/docs = docs/*

View File

@ -64,6 +64,9 @@
when: influxdb_url is defined or influxdb_create_data_file|bool
ignore_errors: true
- include: sova.yml
ignore_errors: true
- name: fetch stackviz results to the root of the collect_dir
shell: >
if [ -d {{ artcl_collect_dir }}/undercloud/var/log/extra/stackviz/data ]; then

17
tasks/sova.yml Normal file
View File

@ -0,0 +1,17 @@
---
- name: Include sova patterns
include_vars:
file: sova-patterns.yml
name: pattern_config
- name: Run sova task
sova:
config: "{{ pattern_config }}"
files:
console: "{{ ansible_user_dir }}/workspace/logs/quickstart_install.log"
errors: "/var/log/errors.txt"
"ironic-conductor": "/var/log/containers/ironic/ironic-conductor.log"
syslog: "/var/log/journal.txt"
logstash: "/var/log/extra/logstash.txt"
result: "{{ ansible_user_dir }}/workspace/logs/failures_file"
result_file_dir: "{{ ansible_user_dir }}/workspace/logs"

677
vars/sova-patterns.yml Normal file
View File

@ -0,0 +1,677 @@
---
regexes:
- regex: 'Killed\s+timeout -s 9 '
name: "timeout_re"
- regex: '1;31mError: .+?\W(\w+)::'
name: "puppet_re"
- regex: 'Could not resolve host: (\S+); Name or service not known'
name: "resolving_re"
- regex: 'mError: (\S+?) \S+ returned 1 instead of one of'
name: "exec_re"
- regex: "Failed to build (.*)"
name: "failed_deps_re"
- regex: 'curl. .*? couldn.t open file "(.*?)"'
name: "curl_re"
- regex: 'fatal: Unable to look up (\S+)'
name: "git_re"
- regex: 'Deployment exited with non-zero status code: (\d+)'
name: "deploy_re"
- regex: 'Error: Could not find data item (\w+) in any Hiera data file and no default supplied'
name: "hiera_re"
- regex: 'mError: /Stage\[main\]/\w+/Exec\[(.+?)\]'
name: "puppetexec_re"
- regex: "Job for (.+) failed because the control process exited with error code."
name: "command_exe"
- regex: "stderr: 'fatal: unable to access 'http.+/devstack-gate/.*Network is unreachable.*"
name: "zcl_re"
- regex: "fatal: unable to access 'http.*Network is unreachable"
name: "gitnet_re"
- regex: "ssh: connect to host .+ port .+: No route to host"
name: "ssh_re"
- regex: "mError: .* at /etc/puppet/modules/([^/]+)/"
name: "pup_module_re"
- regex: 'systemd: (\S+).service failed'
name: "service_fail_re"
- regex: '\[([\w-]+)\]: Failed to call refresh'
name: "fail_refresh_re"
- regex: "Disk volume where .* is located doesn't have enough disk space"
name: "iron_space_re"
- regex: 'overcloud-validate.sh 2>&1 .*"rc": 1'
name: "oooq_validate_fail"
- regex: 'overcloud_image_build_script.sh 2>&1 .*"rc": 1'
name: "oooq_image_build_fail"
- regex: 'undercloud-install.sh 2>&1 .*"rc": 1'
name: "oooq_undercloud_fail"
- regex: '^.*"cmd": "(?:set -o pipefail && )*([^\s]*).*"rc": [^0]+.*\n(?!.*ignoring).*'
name: "oooq_command_fail"
multiline: true
- regex: '^.*"cmd": ".* (tempest_output\.log).*"rc": 1.*\n(?!.*ignoring).*'
name: "oooq_tempest_fail"
- regex: 'Error: Execution of .*yum.* returned 1: Error downloading packages'
name: "yum_install_fail_re"
- regex: 'ERROR! the playbook: .* could not be found'
name: "playbook_err_re"
- regex: 'fatal: .* FAILED! =>.*"msg": .* ([^\s]+) is undefined"'
name: "ansible_und_err"
- regex: "can't read ([^ ]+): No such file or directory"
name: "missing_file"
- regex: ' Could not install requirement ([^\s]+) from'
name: "pip_install_fail"
- regex: 'fatal:.*FAILED! .*undercloud.qcow2.*convert_image.sh'
name: "convert_fail_re"
- regex: 'ERROR:kolla.common.utils:([^\s]+) Failed with status: error'
name: kolla_fail_re
- regex: 'toci_gate_test.sh:.*sudo kill -9 \d+'
name: ovb_kill9_re
- regex: 'Error: .*The (.+) hiera key is undefined'
name: hiera_key_undefined_re
- regex: 'Stack "overcloud" .+ Timed out'
name: stack_oc_timeout_re
- regex: '(\S+): unbound variable'
name: unbound_var_re
- regex: 'Error, some other host \(.+\) already uses address .+'
name: same_host_re
- regex: 'Failed to start (.+) container'
name: failed_container_re
- regex: "msg\": \"(.*) isn't working \\(healthcheck failed\\)"
name: healthcheck_re
patterns:
"console":
- id: 1
logstash: ""
msg: "Overcloud stack installation: SUCCESS."
pattern: "Stack overcloud CREATE_COMPLETE"
tag: "info"
- id: 2
logstash: "Stack overcloud CREATE_FAILED"
msg: "Overcloud stack: FAILED."
pattern: "Stack overcloud CREATE_FAILED"
tag: "info"
- id: 3
logstash: "No valid host was found. There are not enough hosts"
msg: "No valid host was found."
pattern: "No valid host was found. There are not enough hosts"
tag: "code"
- id: 4
logstash: "Failed to connect to trunk.rdoproject.org"
msg: "Connection failure to trunk.rdoproject.org."
pattern: "Failed to connect to trunk.rdoproject.org port 80"
tag: "infra"
- id: 5
logstash: ""
msg: "Overcloud pingtest FAILED."
pattern: "Overloud pingtest, FAIL"
tag: "code"
- id: 6
logstash: ""
msg: "Overcloud pingtest FAILED."
pattern: "Overcloud pingtest, failed"
tag: "code"
- id: 7
logstash: "Error contacting Ironic server: Node"
msg: "Ironic introspection FAIL."
pattern: "Error contacting Ironic server: Node "
tag: "code"
- id: 8
logstash: "Introspection completed with errors:"
msg: "Ironic introspection FAIL."
pattern: "Introspection completed with errors:"
tag: "code"
- id: 9
logstash: ": Introspection timeout"
msg: "Introspection timeout."
pattern: ": Introspection timeout"
tag: "code"
- id: 10
logstash: ""
msg: "Ironic: Host locking error."
pattern: "is locked by host localhost.localdomain, please retry"
tag: "code"
- id: 11
logstash: "Timed out waiting for node "
msg: "Ironic node register FAIL: timeout for node."
pattern: "Timed out waiting for node "
tag: "code"
- id: 12
logstash: "GATE_RETVAL=137"
msg: "Killed by timeout."
pattern: "Killed ./testenv-client -b"
tag: "infra"
- id: 13
logstash: "GATE_RETVAL=137"
msg: "Killed by timeout."
pattern: "timeout_re"
tag: "infra"
- id: 14
logstash: "puppet_re"
msg: "Puppet {} FAIL."
pattern: "puppet_re"
tag: "code"
- id: 15
logstash: "exec_re"
msg: "Program {} FAIL."
pattern: "exec_re"
tag: "code"
- id: 16
logstash: "ERROR:dlrn:"
msg: "Delorean FAIL."
pattern: "ERROR:dlrn:"
tag: "code"
- id: 17
logstash: "500 Internal Server Error: Failed to upload image"
msg: "Glance upload FAIL."
pattern: "500 Internal Server Error: Failed to upload image"
tag: "code"
- id: 18
logstash: ""
msg: "Jenkins slave FAIL."
pattern: "Slave went offline during the build"
tag: "infra"
- id: 19
logstash: "resolving_re"
msg: "DNS resolve of {} FAIL."
pattern: "resolving_re"
tag: "infra"
- id: 20
logstash: "fatal: The remote end hung up unexpectedly"
msg: "Git clone repo FAIL."
pattern: "fatal: The remote end hung up unexpectedly"
tag: "infra"
- id: 23
logstash: "FATAL: no longer a configured node for"
msg: "Slave FAIL: no longer a configured node"
pattern: "FATAL: no longer a configured node for "
tag: "infra"
- id: 24
logstash: "cd: /opt/stack/new/delorean/data/repos: No such file or directory"
msg: "Delorean failed to build the change under review."
pattern: "cd: /opt/stack/new/delorean/data/repos: No such file or directory"
tag: "code"
- id: 25
logstash: "[ERROR] - SEVERE ERROR occurs: java.lang"
msg: "Jenkins slave FAIL: InterruptedException"
pattern: "[ERROR] - SEVERE ERROR occurs: java.lang.InterruptedException"
tag: "infra"
- id: 26
logstash: "Killed bash -xe /opt/stack/new/tripleo-ci/toci_gate_test.sh"
msg: "Main script timeout"
pattern: "Killed bash -xe /opt/stack/new/tripleo-ci/toci_gate_test.sh"
tag: "infra"
- id: 27
logstash: "Command 'instack-install-undercloud' returned non-zero exit status"
msg: "Undercloud install FAIL."
pattern: "Command 'instack-install-undercloud' returned non-zero exit status"
tag: "code"
- id: 28
logstash: "failed_deps_re"
msg: "Failed to build dep {}."
pattern: "failed_deps_re"
tag: "infra"
- id: 29
logstash: "curl_re"
msg: "Failed to upload or get image: {}."
pattern: "curl_re"
tag: "infra"
- id: 30
logstash: "error: command 'gcc' failed with exit status 1"
msg: "Failed to compile deps."
pattern: "error: command 'gcc' failed with exit status 1"
tag: "infra"
- id: 31
logstash: "crm_resource for openstack"
msg: "'crm_resource' check failed because timeout."
pattern: "crm_resource for openstack"
tag: "infra"
- id: 32
logstash: "failed to open 'instack.qcow2': No such file or"
msg: "FAIL to build image instack.qcow2."
pattern: "failed to open 'instack.qcow2': No such file or"
tag: "code"
- id: 33
logstash: "Stack overcloud UPDATE_FAILED"
msg: "Stack update FAILED."
pattern: "Stack overcloud UPDATE_FAILED"
tag: "info"
- id: 34
logstash: "git_re"
msg: "DNS resolve of {} FAIL."
pattern: "git_re"
tag: "infra"
- id: 35
logstash: "deploy_re"
msg: "Deployment exited with code {}."
pattern: "deploy_re"
tag: "code"
- id: 36
logstash: "No connected Gearman servers"
msg: "Gearman problem."
pattern: "Error: No connected Gearman servers"
tag: "infra"
- id: 37
logstash: "Overcloud pingtest FAILED"
msg: "Overcloud pingtest FAILED."
pattern: "Overcloud pingtest FAILED"
tag: "code"
- id: 38
logstash: "... FAILED"
msg: "Tempest tests FAILED."
pattern: "... FAILED"
tag: "code"
- id: 39
logstash: "No more mirrors to try"
msg: "Network issue 'No more mirrors'."
pattern: "No more mirrors to try"
tag: "infra"
- id: 40
logstash: "The gearman Job has failed"
msg: "Gearman task FAILED."
pattern: "ERROR - The gearman Job has failed"
tag: "infra"
- id: 41
logstash: "ERROR - Couldn't retrieve env"
msg: "OVB Environment setup FAILED."
pattern: "ERROR - Couldn't retrieve env"
tag: "infra"
- id: 42
logstash: 'Command "python setup.py egg_info" failed with error code 1'
msg: "Pip install FAIL."
pattern: 'Command "python setup.py egg_info" failed with error code 1'
tag: "infra"
- id: 43
logstash: "MessagingTimeout: Timed out waiting for a reply to message ID"
msg: "Message timeout."
pattern: "MessagingTimeout: Timed out waiting for a reply to message ID"
tag: "code"
- id: 44
logstash: "504 Gateway Time-out: The server didn't respond in time"
msg: "Gateway timeout 504."
pattern: "504 Gateway Time-out: The server didn't respond in time"
tag: "infra"
- id: 45
logstash: "Exception registering nodes:"
msg: "Node registration FAIL."
pattern: "Exception registering nodes:"
tag: "code"
- id: 46
logstash: "400 Bad Request: Client disconnected before sending all data to backend (HTTP 400)"
msg: "HTTP 400 Error."
pattern: "400 Bad Request: Client disconnected before sending all data to backend (HTTP 400)"
tag: "code"
- id: 47
logstash: "command_exe"
msg: "{} FAIL."
pattern: "command_exe"
tag: "code"
- id: 48
logstash: "Network is unreachable"
msg: "Zuul-cloner network FAIL."
pattern: "zcl_re"
tag: "infra"
- id: 49
logstash: "Network is unreachable"
msg: "Network FAIL."
pattern: "gitnet_re"
tag: "infra"
- id: 50
logstash: "port 22: No route to host"
msg: "SSH to host FAIL."
pattern: "ssh_re"
tag: "code"
- id: 51
logstash: "CommandError: No image with a name or ID of"
msg: "No image on the host."
pattern: "CommandError: No image with a name or ID of"
tag: "code"
- id: 52
logstash: "Not enough nodes - available"
msg: "Not enough nodes are available."
pattern: "Not enough nodes - available"
tag: "code"
- id: 53
logstash: "Timing out after 300 seconds:"
msg: "Pingtest stack timeout."
pattern: "Timing out after 300 seconds:"
tag: "code"
- id: 54
logstash: "504 Gateway Time-out"
msg: "504 Gateway Time-out."
pattern: "504 Gateway Time-out"
tag: "code"
- id: 55
logstash: "Gateway Time-out (HTTP 504)"
msg: "504 Gateway Time-out."
pattern: "Gateway Time-out (HTTP 504)"
tag: "code"
- id: 56
logstash: "Retrying (Retry(total=0, connect=None, read=None, redirect=None)) after connection broken by"
msg: "Pip networking timeout."
pattern: "Retrying (Retry(total=0, connect=None, read=None, redirect=None)) after connection broken by"
tag: "infra"
- id: 57
logstash: "Could not evaluate: Cannot allocate memory - fork"
msg: "Puppet memory fail."
pattern: "Could not evaluate: Cannot allocate memory - fork(2)"
tag: "infra"
- id: 58
logstash: "Exception introspecting nodes"
msg: "Introspection FAIL."
pattern: "Exception introspecting nodes"
tag: "code"
- id: 59
logstash: ""
msg: "Overcloud pingtest FAILED."
pattern: "oooq_validate_fail"
tag: "code"
- id: 59
logstash: ""
msg: "Overcloud pingtest FAILED."
pattern: "oooq_validate_fail"
tag: "code"
- id: 60
logstash: ""
msg: "{} fail."
pattern: "oooq_command_fail"
tag: "code"
- id: 61
logstash: '"overcloud_deploy_result": "failed"'
msg: "Overcloud stack: FAILED."
pattern: '"overcloud_deploy_result": "failed"'
tag: "info"
- id: 62
logstash: "oooq_image_build_fail"
msg: "FAIL to build image."
pattern: "oooq_image_build_fail"
tag: "code"
- id: 63
logstash: "oooq_undercloud_fail"
msg: "Undercloud install FAIL."
pattern: "oooq_undercloud_fail"
tag: "code"
- id: 64
logstash: "setup script run by this job failed - exit code: 143"
msg: "Forced to stop."
pattern: "setup script run by this job failed - exit code: 143"
tag: "infra"
- id: 65
msg: "Tempest test failure."
pattern: "oooq_tempest_fail"
tag: "code"
logstash: "oooq_tempest_fail"
- id: 66
msg: 'Image Build Error'
pattern: '*** Image Build Error ***'
tag: "infra"
logstash: ""
- id: 67
msg: "Failed to find playbook"
pattern: "playbook_err_re"
tag: "infra"
logstash: ""
- id: 68
msg: "Ansible undefined variable {}"
pattern: "ansible_und_err"
tag: "infra"
logstash: "{} is undefined"
- id: 69
msg: "Multinode SSH connection failed."
pattern: "Failed to connect to the host via ssh: ssh: connect to host"
tag: "infra"
logstash: "Failed to connect to the host via ssh: ssh: connect to host"
- id: 70
msg: "Overcloud update FAIL."
pattern: "Overcloud update - FAILED"
tag: "code"
logstash: "Overcloud update - FAILED"
- id: 71
msg: "Heat property error."
pattern: "ERROR: Property error:"
tag: "code"
logstash: "ERROR: Property error:"
- id: 72
msg: "OVB Environment setup FAILED."
pattern: "testenv-client - INFO - Received job : Failed creating OVB stack"
tag: "infra"
logstash: "testenv-client - INFO - Received job : Failed creating OVB stack"
- id: 73
msg: "{} file missing."
pattern: "missing_file"
tag: "infra"
logstash: "can't read {}: No such file or directory"
- id: 74
msg: "Pip install FAIL."
pattern: "pip_install_fail"
tag: "infra"
logstash: "| Could not install requirement"
- id: 75
msg: "Hardlink copy FAIL."
pattern: "Invalid cross-device link"
tag: "infra"
logstash: "Invalid cross-device link"
- id: 76
msg: "Forced to stop."
pattern: ": exit_value=143"
tag: "infra"
logstash: ": exit_value=143"
- id: 77
msg: "Image convert to undercloud FAILED."
pattern: "convert_fail_re"
tag: "code"
logstash: ""
- id: 78
msg: "Containers build FAILED."
pattern: 'fatal: [localhost]: FAILED! => {"changed": true, "cmd": ["kolla-build"'
tag: "infra"
logstash: ""
- id: 79
msg: "Container {} FAILED to build."
pattern: 'kolla_fail_re'
tag: "infra"
logstash: "ERROR:kolla.common.utils:{} Failed with status: error"
- id: 80
msg: "Websocket timeout"
pattern: "The Workflow finished successfully but no messages were received before the WebSocket timed out"
tag: "code"
logstash: "The Workflow finished successfully but no messages were received before the WebSocket timed out"
- id: 81
msg: "Timeout to get OVB stack from te-broker"
pattern: "ovb_kill9_re"
tag: "infra"
logstash: "sudo kill -9"
- id: 82
msg: 'No such file in container.'
pattern: 'exec user process caused "no such file or directory'
logstash: 'exec user process caused "no such file or directory'
tag: 'code'
- id: 83
msg: "Job timeout."
pattern: "RUN END RESULT_TIMED_OUT:"
tag: "infra"
logstash: "RUN END RESULT_TIMED_OUT:"
- id: 84
msg: "Timeout to create overcloud stack."
pattern: "stack_oc_timeout_re"
tag: "infra"
logstash: ""
- id: 85
msg: "Failed to create OVB stack."
pattern: "RuntimeError: Failed to create stack baremetal"
tag: "infra"
logstash: ""
- id: 86
msg: "Unbound variable {}."
pattern: 'unbound_var_re'
tag: "code"
logstash: ": unbound variable"
"errors":
- id: 200
logstash: 'Image prepare failed: Pulling image failed:'
msg: "Buildah pull image failed."
pattern: 'Image prepare failed: Pulling image failed: cmd "buildah'
tag: "infra"
- id: 201
logstash: '"code":"UNAUTHORIZED","message":"authentication required"'
msg: "Unauthorized error from container registry."
pattern: '"code":"UNAUTHORIZED","message":"authentication required"'
tag: "infra"
"ironic-conductor":
- id: 300
logstash: "Timeout reached while waiting for callback for node"
msg: "Ironic deployment timeout."
pattern: "Timeout reached while waiting for callback for node"
tag: "code"
- id: 301
logstash: "is located doesn't have enough disk space. Required"
msg: "No space on disk for Ironic."
pattern: "iron_space_re"
tag: "infra"
"syslog":
- exclude:
- "glean@"
- "docker-storage-setup"
id: 400
logstash: ""
msg: "{} service FAIL."
pattern: "service_fail_re"
tag: "command_exe"
- id: 401
logstash: "Failed to start"
msg: "Failed to start {} container"
pattern: "failed_container_re"
tag: "infra"
"logstash":
- id: 500
logstash: " is not pingable."
msg: "Ping timeout when deploying OC."
pattern: " is not pingable."
tag: "infra"
- id: 501
logstash: ""
msg: "Yum download network failure."
pattern: "yum_install_fail_re"
tag: "infra"
- id: 502
pattern: "UnixHTTPConnectionPool(host='localhost', port=None): Read timed out"
msg: "Container upload timed out."
tag: "infra"
logstash: "UnixHTTPConnectionPool(host='localhost', port=None): Read timed out"
- id: 503
pattern: '404 Client Error: Not Found ("no such id: docker.io'
msg: 'Container image not found.'
tag: "infra"
logstash: '404 Client Error: Not Found ("no such id:'
- id: 504
logstash: ""
msg: "Failed to connect to endpoints of overcloud."
pattern: "Failed to contact the endpoint at http"
tag: "code"
- id: 505
logstash: 'Ansible failed, check log at /var/lib/mistral/overcloud/ansible.log'
msg: 'Overcloud deploy failed.'
tag: 'code'
pattern: 'Ansible failed, check log at /var/lib/mistral/overcloud/ansible.log'
- id: 506
logstash: ""
msg: 'Overcloud image create failed.'
tag: 'code'
pattern: "CalledProcessError: Command '['disk-image-create'"
- id: 507
logstash: ""
msg: 'Container not found.'
tag: 'infra'
pattern: "ImageNotFoundException: Not found image: docker:"
- id: 508
logstash: ""
msg: 'Ironic node provision failed.'
tag: 'infra'
pattern: "error: Failed to prepare node"
- id: 509
logstash: ""
msg: 'IPMI to nodes failed.'
tag: 'infra'
pattern: "Failed to prepare to deploy: IPMI call failed:"
- id: 510
logstash: ""
msg: "Hiera key {} is undefined."
pattern: "hiera_key_undefined_re"
tag: "code"
- id: 511
logstash: "Found ansible errors for undercloud deployment"
msg: "Undercloud install failed."
pattern: "Found ansible errors for undercloud deployment"
tag: "code"
- id: 512
logstash: ""
msg: "IPMI to nodes failed."
pattern: "Error: IPMI call failed: power status"
tag: "infra"
- id: 513
logstash: ""
msg: "Nova failure: no valid host was found."
pattern: 'ResourceInError: resources.NovaCompute: Went to status ERROR due to "Message: No valid host was found. , Code: 500'
tag: "infra"
- id: 514
logstash: "DeploymentError: Stack create failed"
msg: "Standalone or Undercloud Heat stack failed."
pattern: "DeploymentError: Stack create failed"
tag: "code"
- id: 515
logstash: "puppet_re"
msg: "Puppet {} FAIL."
pattern: "puppet_re"
tag: "code"
- id: 516
logstash: "Can't connect to local MySQL server through socket"
msg: "MySQL failure."
pattern: "Can't connect to local MySQL server through socket"
tag: "code"
- id: 517
logstash: "Could not evaluate: Cannot allocate memory - fork"
msg: "Puppet memory fail."
pattern: "Could not evaluate: Cannot allocate memory - fork(2)"
tag: "infra"
- id: 518
logstash: "hiera_re"
msg: "No {} in Hiera."
pattern: "hiera_re"
tag: "code"
- id: 519
logstash: "hiera_re"
msg: "No {} in Hiera."
pattern: "hiera_re"
tag: "code"
- id: 520
logstash: "puppetexec_re"
msg: "{} FAIL."
pattern: "puppetexec_re"
tag: "code"
- id: 521
logstash: "pup_module_re"
msg: "Puppet module '{}' FAIL."
pattern: "pup_module_re"
tag: "code"
- id: 522
logstash: "fail_refresh_re"
msg: "{} FAIL."
pattern: "fail_refresh_re"
tag: "code"
- id: 523
logstash: "already uses address"
msg: "MAC conflict between hosts."
pattern: "same_host_re"
tag: "infra"
- id: 524
logstash: ' ... FAILED'
msg: 'Tempest test failed.'
tag: "code"
pattern: ' ... FAILED'
- id: 525
logstash: "healthcheck failed"
msg: "Failed healthcheck of {}."
pattern: "healthcheck_re"
tag: "code"
"bmc":
- id: 600
logstash: 'socket.error: [Errno 99] Cannot assign requested address'
msg: 'Introspection failed, cannot get IP address'
tag: 'infra'
pattern: 'socket.error: [Errno 99] Cannot assign requested address'