Transform artcl into a collection
Because Ansible official testing tools (ansible-test) cannot be used without a collection, we change the code layout to make conformant. WARNING: The role is no longer considered to be named "ansible-role-collect-logs" but "collect_logs" instead, with a temporary alias called "collect-logs". Checklist: - [x] ansible-test sanity checks runs (does not need to pass) - [x] zuul is still able to use the role - [x] infrared is still able to use the role - [x] molecule tests are running and passing - [x] tripleo-ci jobs still collect the files One symlink is still needed for infrared until related patch lands: https://review.gerrithub.io/c/redhat-openstack/infrared/+/508861 Change-Id: Ib87622797a284d837ee579d9cccec0ed73306626 Story: TRIPLEOCI-305
This commit is contained in:
parent
dd904e9518
commit
1c71e5098f
5
MANIFEST.in
Normal file
5
MANIFEST.in
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
global-exclude __pycache__
|
||||||
|
exclude infrared_plugin
|
||||||
|
exclude zuul.d
|
||||||
|
exclude test-playbooks
|
||||||
|
exclude plugins
|
@ -1,4 +1,4 @@
|
|||||||
collect-logs
|
collect_logs
|
||||||
============
|
============
|
||||||
|
|
||||||
Ansible role for aggregating logs from different nodes.
|
Ansible role for aggregating logs from different nodes.
|
||||||
@ -221,7 +221,7 @@ Example Role Playbook
|
|||||||
- name: Gather logs
|
- name: Gather logs
|
||||||
hosts: all:!localhost
|
hosts: all:!localhost
|
||||||
roles:
|
roles:
|
||||||
- collect-logs
|
- collect_logs
|
||||||
|
|
||||||
** Note:
|
** Note:
|
||||||
The tasks that collect data from the nodes are executed with ignore_errors.
|
The tasks that collect data from the nodes are executed with ignore_errors.
|
||||||
|
@ -6,3 +6,8 @@ callback_whitelist = profile_tasks
|
|||||||
|
|
||||||
# Attempt to load custom modules whether it's installed system-wide or from a virtual environment
|
# Attempt to load custom modules whether it's installed system-wide or from a virtual environment
|
||||||
roles_path = roles:$VIRTUAL_ENV/share/ansible/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:$VIRTUAL_ENV/usr/share/ansible/roles
|
roles_path = roles:$VIRTUAL_ENV/share/ansible/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:$VIRTUAL_ENV/usr/share/ansible/roles
|
||||||
|
|
||||||
|
# Required by infrared
|
||||||
|
host_key_checking = False
|
||||||
|
forks = 500
|
||||||
|
timeout = 300
|
||||||
|
25
galaxy.yml
Normal file
25
galaxy.yml
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
---
|
||||||
|
name: collect_logs
|
||||||
|
namespace: tripleo
|
||||||
|
version: 0.0.1
|
||||||
|
authors:
|
||||||
|
- tripleo
|
||||||
|
readme: README.rst
|
||||||
|
|
||||||
|
build_ignore:
|
||||||
|
- .ansible
|
||||||
|
- .cache
|
||||||
|
- .github
|
||||||
|
- .gitignore
|
||||||
|
- .pytest_cache
|
||||||
|
- .vscode
|
||||||
|
- .tox
|
||||||
|
- dist
|
||||||
|
- tox.ini
|
||||||
|
- .eggs
|
||||||
|
- .mypy_cache
|
||||||
|
- "*.egg-info"
|
||||||
|
- modules
|
||||||
|
- module_utils
|
||||||
|
- infrared_plugin
|
||||||
|
- scripts
|
@ -24,7 +24,7 @@
|
|||||||
|
|
||||||
- name: Ansible role collect logs
|
- name: Ansible role collect logs
|
||||||
include_role:
|
include_role:
|
||||||
name: ansible-role-collect-logs
|
name: collect_logs
|
||||||
|
|
||||||
# This section takes care of preparing the collected data for publishing
|
# This section takes care of preparing the collected data for publishing
|
||||||
# and for publishing itself
|
# and for publishing itself
|
||||||
@ -65,7 +65,7 @@
|
|||||||
|
|
||||||
- name: Ansible role collect logs
|
- name: Ansible role collect logs
|
||||||
include_role:
|
include_role:
|
||||||
name: ansible-role-collect-logs
|
name: collect_logs
|
||||||
when: artcl_publish|default(false)|bool
|
when: artcl_publish|default(false)|bool
|
||||||
|
|
||||||
- name: Delete artifact files from localhost
|
- name: Delete artifact files from localhost
|
||||||
|
@ -3,7 +3,7 @@
|
|||||||
config:
|
config:
|
||||||
plugin_type: other
|
plugin_type: other
|
||||||
entry_point: main.yml
|
entry_point: main.yml
|
||||||
roles_path: ../
|
roles_path: ../roles/
|
||||||
subparsers:
|
subparsers:
|
||||||
ansible-role-collect-logs:
|
ansible-role-collect-logs:
|
||||||
description: An Ansible role for aggregating logs from different nodes.
|
description: An Ansible role for aggregating logs from different nodes.
|
||||||
|
1
infrared_plugin/roles
Symbolic link
1
infrared_plugin/roles
Symbolic link
@ -0,0 +1 @@
|
|||||||
|
../roles
|
@ -1,2 +1,3 @@
|
|||||||
pbr>=1.6
|
pbr>=1.6
|
||||||
ansible>=2.5,<2.10
|
# Do not remove 2.10, ansible-test and tox-ansible require it
|
||||||
|
ansible>=2.5,<2.11
|
||||||
|
1
roles/collect_logs/library
Symbolic link
1
roles/collect_logs/library
Symbolic link
@ -0,0 +1 @@
|
|||||||
|
../../plugins/modules
|
102
roles/collect_logs/module_utils/sova_lib.py
Normal file
102
roles/collect_logs/module_utils/sova_lib.py
Normal file
@ -0,0 +1,102 @@
|
|||||||
|
#!/usr/bin/python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import gzip
|
||||||
|
import logging
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
try:
|
||||||
|
import regex as regex_module
|
||||||
|
except ImportError:
|
||||||
|
import re as regex_module
|
||||||
|
|
||||||
|
|
||||||
|
logging.basicConfig(
|
||||||
|
format=('%(asctime)s - %(name)s - %(levelname)s - '
|
||||||
|
'%(module)s.%(funcName)s:%(lineno)d - %(message)s'))
|
||||||
|
log = logging.getLogger('parser')
|
||||||
|
log.setLevel(logging.ERROR)
|
||||||
|
|
||||||
|
|
||||||
|
class Pattern(object):
|
||||||
|
def __init__(self, data):
|
||||||
|
self.data = data
|
||||||
|
self.load_yaml()
|
||||||
|
self.setup_regexes()
|
||||||
|
self.setup_patterns()
|
||||||
|
|
||||||
|
def load_yaml(self):
|
||||||
|
if isinstance(self.data, dict):
|
||||||
|
self.config = self.data
|
||||||
|
else:
|
||||||
|
self.config = yaml.safe_load(self.data)
|
||||||
|
|
||||||
|
def setup_regexes(self):
|
||||||
|
self.regexes = {}
|
||||||
|
if self.config:
|
||||||
|
for regexp in self.config.get('regexes', []):
|
||||||
|
flags = []
|
||||||
|
if regexp.get('multiline'):
|
||||||
|
flags.append(regex_module.MULTILINE)
|
||||||
|
self.regexes[regexp.get('name')] = regex_module.compile(
|
||||||
|
r'{}'.format(regexp.get('regex')), *flags)
|
||||||
|
|
||||||
|
def setup_patterns(self):
|
||||||
|
self._patterns = self.config.get('patterns', {})
|
||||||
|
if self._patterns:
|
||||||
|
for key in self._patterns:
|
||||||
|
for p in self._patterns[key]:
|
||||||
|
if p['pattern'] in self.regexes:
|
||||||
|
p['pattern'] = self.regexes[p['pattern']]
|
||||||
|
if p['logstash'] in self.regexes:
|
||||||
|
p['logstash'] = self.regexes[p['logstash']]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def patterns(self):
|
||||||
|
return self._patterns
|
||||||
|
|
||||||
|
|
||||||
|
def line_match(pat, line, exclude=None):
|
||||||
|
if isinstance(pat, str):
|
||||||
|
return pat in line
|
||||||
|
found = pat.search(line)
|
||||||
|
if not found:
|
||||||
|
return False
|
||||||
|
if found.groups():
|
||||||
|
if exclude:
|
||||||
|
if any([i in found.group(1) for i in exclude]):
|
||||||
|
return False
|
||||||
|
return found.group(1)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def parse(text_file, patterns):
|
||||||
|
ids = []
|
||||||
|
msgs = []
|
||||||
|
if text_file.split(".")[-1] == "gz":
|
||||||
|
open_func = gzip.open
|
||||||
|
else:
|
||||||
|
open_func = open
|
||||||
|
with open_func(text_file, "rt") as finput:
|
||||||
|
text = finput.read()
|
||||||
|
for p in patterns:
|
||||||
|
line_matched = line_match(
|
||||||
|
p["pattern"], text, exclude=p.get("exclude"))
|
||||||
|
if line_matched:
|
||||||
|
log.debug("Found pattern {} in file {}".format(
|
||||||
|
repr(p), text_file))
|
||||||
|
ids.append(p["id"])
|
||||||
|
msgs.append(p["msg"].format(line_matched))
|
||||||
|
return list(set(ids)), list(set(msgs))
|
@ -37,7 +37,7 @@
|
|||||||
# brief call used a very short override artcl_commands, enough to validate
|
# brief call used a very short override artcl_commands, enough to validate
|
||||||
# that the combining of the commands works. Later we import the role with
|
# that the combining of the commands works. Later we import the role with
|
||||||
# its default artcl_commands in order to test these commands, too.
|
# its default artcl_commands in order to test these commands, too.
|
||||||
- name: "Include ansible-role-collect-logs :: collect (brief)"
|
- name: "Include collect_logs :: collect (brief)"
|
||||||
vars:
|
vars:
|
||||||
artcl_collect: true
|
artcl_collect: true
|
||||||
artcl_commands:
|
artcl_commands:
|
||||||
@ -52,7 +52,7 @@
|
|||||||
cmd: cat /proc/swaps
|
cmd: cat /proc/swaps
|
||||||
capture_file: /var/log/extra/swaps.txt
|
capture_file: /var/log/extra/swaps.txt
|
||||||
include_role:
|
include_role:
|
||||||
name: "ansible-role-collect-logs"
|
name: collect_logs
|
||||||
|
|
||||||
- name: Verify expected combined commands
|
- name: Verify expected combined commands
|
||||||
assert:
|
assert:
|
||||||
@ -73,7 +73,7 @@
|
|||||||
vars:
|
vars:
|
||||||
artcl_collect: true
|
artcl_collect: true
|
||||||
include_role:
|
include_role:
|
||||||
name: "ansible-role-collect-logs"
|
name: collect_logs
|
||||||
|
|
||||||
- name: "Converge publish play"
|
- name: "Converge publish play"
|
||||||
hosts: localhost
|
hosts: localhost
|
||||||
@ -86,7 +86,7 @@
|
|||||||
artcl_collect: false
|
artcl_collect: false
|
||||||
artcl_publish: true
|
artcl_publish: true
|
||||||
include_role:
|
include_role:
|
||||||
name: "ansible-role-collect-logs"
|
name: collect_logs
|
||||||
|
|
||||||
- debug:
|
- debug:
|
||||||
msg: |
|
msg: |
|
@ -33,7 +33,7 @@
|
|||||||
|
|
||||||
- name: "Copy ansible-role-collect-logs to test host"
|
- name: "Copy ansible-role-collect-logs to test host"
|
||||||
synchronize:
|
synchronize:
|
||||||
src: "{{ playbook_dir }}/../../"
|
src: "{{ playbook_dir }}/../../../../"
|
||||||
dest: "{{ ansible_env.HOME }}/artcl-src"
|
dest: "{{ ansible_env.HOME }}/artcl-src"
|
||||||
rsync_opts:
|
rsync_opts:
|
||||||
- "--exclude=.tox"
|
- "--exclude=.tox"
|
@ -2,9 +2,9 @@
|
|||||||
- name: Converge
|
- name: Converge
|
||||||
hosts: all
|
hosts: all
|
||||||
tasks:
|
tasks:
|
||||||
- name: "Include ansible-role-collect-logs"
|
- name: Include collect_logs
|
||||||
include_role:
|
include_role:
|
||||||
name: "ansible-role-collect-logs"
|
name: collect_logs
|
||||||
tasks_from: sova.yml
|
tasks_from: sova.yml
|
||||||
tags:
|
tags:
|
||||||
- molecule-idempotence-notest
|
- molecule-idempotence-notest
|
14
setup.cfg
14
setup.cfg
@ -19,17 +19,11 @@ setup-hooks =
|
|||||||
pbr.hooks.setup_hook
|
pbr.hooks.setup_hook
|
||||||
|
|
||||||
[files]
|
[files]
|
||||||
|
# Allows us to install the role using pip so Ansible can find it.
|
||||||
data_files =
|
data_files =
|
||||||
share/ansible/roles/collect-logs/defaults = defaults/*
|
share/ansible/roles/collect-logs = roles/collect_logs/*
|
||||||
share/ansible/roles/collect-logs/meta = meta/*
|
share/ansible/roles/collect-logs/library = plugins/modules/*
|
||||||
share/ansible/roles/collect-logs/tasks = tasks/*
|
share/ansible/roles/collect-logs/module_utils = plugins/module_utils/*
|
||||||
share/ansible/roles/collect-logs/templates = templates/*
|
|
||||||
share/ansible/roles/collect-logs/files = files/*
|
|
||||||
share/ansible/roles/collect-logs/filter_plugins = filter_plugins/*
|
|
||||||
share/ansible/roles/collect-logs/library = library/*
|
|
||||||
share/ansible/roles/collect-logs/module_utils = module_utils/*
|
|
||||||
share/ansible/roles/collect-logs/vars = vars/*
|
|
||||||
share/ansible/roles/collect-logs/scripts = scripts/*
|
|
||||||
share/ansible/roles/collect-logs/docs = docs/*
|
share/ansible/roles/collect-logs/docs = docs/*
|
||||||
|
|
||||||
[wheel]
|
[wheel]
|
||||||
|
@ -2,5 +2,8 @@
|
|||||||
- hosts: all
|
- hosts: all
|
||||||
tasks:
|
tasks:
|
||||||
- name: include ansible-role-collect-logs role
|
- name: include ansible-role-collect-logs role
|
||||||
|
vars:
|
||||||
|
artcl_collect: true
|
||||||
|
artcl_publish: true
|
||||||
include_role:
|
include_role:
|
||||||
name: collect-logs
|
name: collect_logs
|
||||||
|
@ -1,5 +1,6 @@
|
|||||||
import pytest # noqa
|
import pytest # noqa
|
||||||
import flatten_nested_dict
|
import os
|
||||||
|
import sys
|
||||||
from common.utils import (
|
from common.utils import (
|
||||||
AnsibleExitJson, AnsibleFailJson, ModuleTestCase, set_module_args,
|
AnsibleExitJson, AnsibleFailJson, ModuleTestCase, set_module_args,
|
||||||
)
|
)
|
||||||
@ -22,6 +23,12 @@ data:
|
|||||||
group: system
|
group: system
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
# Temporary hack until we adopt official ansible-test unit-testing
|
||||||
|
dir = os.path.join(os.path.dirname(__file__), "../roles/collect_logs/library")
|
||||||
|
sys.path.append(dir)
|
||||||
|
print(dir)
|
||||||
|
import flatten_nested_dict # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
class TestFlattenNestedDict(ModuleTestCase):
|
class TestFlattenNestedDict(ModuleTestCase):
|
||||||
|
|
||||||
|
4
tox.ini
4
tox.ini
@ -3,6 +3,8 @@
|
|||||||
minversion = 3.4.0
|
minversion = 3.4.0
|
||||||
envlist = docs, linters, molecule
|
envlist = docs, linters, molecule
|
||||||
skipdist = True
|
skipdist = True
|
||||||
|
requires =
|
||||||
|
tox-ansible >= 1.0.3
|
||||||
|
|
||||||
[testenv]
|
[testenv]
|
||||||
usedevelop = True
|
usedevelop = True
|
||||||
@ -58,7 +60,7 @@ setenv =
|
|||||||
deps =
|
deps =
|
||||||
ansible>=2.9,<2.10
|
ansible>=2.9,<2.10
|
||||||
molecule[test,docker]>=3.2.2,<3.3 # MIT
|
molecule[test,docker]>=3.2.2,<3.3 # MIT
|
||||||
pytest-molecule
|
pytest-molecule>=1.3.4
|
||||||
pytest-plus # provides support for PYTEST_REQPASS
|
pytest-plus # provides support for PYTEST_REQPASS
|
||||||
commands =
|
commands =
|
||||||
python -m pytest --color=yes --html={envlogdir}/reports.html --self-contained-html {tty:-s} {posargs}
|
python -m pytest --color=yes --html={envlogdir}/reports.html --self-contained-html {tty:-s} {posargs}
|
||||||
|
@ -1,4 +1,13 @@
|
|||||||
---
|
---
|
||||||
|
- job:
|
||||||
|
name: tox-ansible-test-sanity
|
||||||
|
description: Runs ansible-test sanity (tox -e sanity)
|
||||||
|
parent: tox
|
||||||
|
vars:
|
||||||
|
tox_envlist: sanity # dynamic tox env added by tox-ansible
|
||||||
|
# we want to run sanity only on py36 instead of implicit 2.6-3.9 range
|
||||||
|
tox_extra_args: -- --python 3.6
|
||||||
|
|
||||||
- job:
|
- job:
|
||||||
name: zuul-ansible-role-collect-logs
|
name: zuul-ansible-role-collect-logs
|
||||||
description: Validate that zuul can use that role.
|
description: Validate that zuul can use that role.
|
||||||
@ -6,7 +15,6 @@
|
|||||||
run: test-playbooks/zuul-ansible-role-collect-logs.yaml
|
run: test-playbooks/zuul-ansible-role-collect-logs.yaml
|
||||||
roles:
|
roles:
|
||||||
- zuul: opendev.org/openstack/ansible-role-collect-logs
|
- zuul: opendev.org/openstack/ansible-role-collect-logs
|
||||||
name: collect-logs
|
|
||||||
irrelevant-files:
|
irrelevant-files:
|
||||||
- ^vars/sova-patterns.yml$
|
- ^vars/sova-patterns.yml$
|
||||||
|
|
||||||
@ -19,6 +27,8 @@
|
|||||||
jobs: &jobs
|
jobs: &jobs
|
||||||
- openstack-tox-linters
|
- openstack-tox-linters
|
||||||
- openstack-tox-molecule
|
- openstack-tox-molecule
|
||||||
|
- tox-ansible-test-sanity:
|
||||||
|
voting: false # until we fix reported errors
|
||||||
# Limit the number of jobs executed while still assuring a relevant
|
# Limit the number of jobs executed while still assuring a relevant
|
||||||
# level of coverage. If specific tasks are to be tested we should
|
# level of coverage. If specific tasks are to be tested we should
|
||||||
# consider implementing functional tests for them, especially as
|
# consider implementing functional tests for them, especially as
|
||||||
|
Loading…
Reference in New Issue
Block a user