diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..0abd561 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,5 @@ +global-exclude __pycache__ +exclude infrared_plugin +exclude zuul.d +exclude test-playbooks +exclude plugins diff --git a/README.rst b/README.rst index ad18a64..7f20271 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -collect-logs +collect_logs ============ Ansible role for aggregating logs from different nodes. @@ -221,7 +221,7 @@ Example Role Playbook - name: Gather logs hosts: all:!localhost roles: - - collect-logs + - collect_logs ** Note: The tasks that collect data from the nodes are executed with ignore_errors. diff --git a/ansible.cfg b/ansible.cfg index 63179df..aa05284 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -6,3 +6,8 @@ callback_whitelist = profile_tasks # Attempt to load custom modules whether it's installed system-wide or from a virtual environment roles_path = roles:$VIRTUAL_ENV/share/ansible/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:$VIRTUAL_ENV/usr/share/ansible/roles + +# Required by infrared +host_key_checking = False +forks = 500 +timeout = 300 diff --git a/galaxy.yml b/galaxy.yml new file mode 100644 index 0000000..6a51824 --- /dev/null +++ b/galaxy.yml @@ -0,0 +1,25 @@ +--- +name: collect_logs +namespace: tripleo +version: 0.0.1 +authors: + - tripleo +readme: README.rst + +build_ignore: + - .ansible + - .cache + - .github + - .gitignore + - .pytest_cache + - .vscode + - .tox + - dist + - tox.ini + - .eggs + - .mypy_cache + - "*.egg-info" + - modules + - module_utils + - infrared_plugin + - scripts diff --git a/infrared_plugin/main.yml b/infrared_plugin/main.yml index 825e8a0..8cdcfa9 100644 --- a/infrared_plugin/main.yml +++ b/infrared_plugin/main.yml @@ -24,7 +24,7 @@ - name: Ansible role collect logs include_role: - name: ansible-role-collect-logs + name: collect_logs # This section takes care of preparing the collected data for publishing # and for publishing itself @@ -65,7 +65,7 @@ - name: Ansible role collect logs include_role: - name: ansible-role-collect-logs + name: collect_logs when: artcl_publish|default(false)|bool - name: Delete artifact files from localhost diff --git a/infrared_plugin/plugin.spec b/infrared_plugin/plugin.spec index 3ba8500..e646d1b 100644 --- a/infrared_plugin/plugin.spec +++ b/infrared_plugin/plugin.spec @@ -3,7 +3,7 @@ config: plugin_type: other entry_point: main.yml - roles_path: ../ + roles_path: ../roles/ subparsers: ansible-role-collect-logs: description: An Ansible role for aggregating logs from different nodes. diff --git a/infrared_plugin/roles b/infrared_plugin/roles new file mode 120000 index 0000000..d8c4472 --- /dev/null +++ b/infrared_plugin/roles @@ -0,0 +1 @@ +../roles \ No newline at end of file diff --git a/module_utils/sova_lib.py b/plugins/module_utils/sova_lib.py similarity index 100% rename from module_utils/sova_lib.py rename to plugins/module_utils/sova_lib.py diff --git a/library/ara_graphite.py b/plugins/modules/ara_graphite.py similarity index 100% rename from library/ara_graphite.py rename to plugins/modules/ara_graphite.py diff --git a/library/ara_influxdb.py b/plugins/modules/ara_influxdb.py similarity index 100% rename from library/ara_influxdb.py rename to plugins/modules/ara_influxdb.py diff --git a/library/flatten_nested_dict.py b/plugins/modules/flatten_nested_dict.py similarity index 100% rename from library/flatten_nested_dict.py rename to plugins/modules/flatten_nested_dict.py diff --git a/library/sova.py b/plugins/modules/sova.py similarity index 100% rename from library/sova.py rename to plugins/modules/sova.py diff --git a/requirements.txt b/requirements.txt index 1545424..e0375c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ pbr>=1.6 -ansible>=2.5,<2.10 +# Do not remove 2.10, ansible-test and tox-ansible require it +ansible>=2.5,<2.11 diff --git a/defaults/main.yml b/roles/collect_logs/defaults/main.yml similarity index 100% rename from defaults/main.yml rename to roles/collect_logs/defaults/main.yml diff --git a/files/collect-container-logs.sh b/roles/collect_logs/files/collect-container-logs.sh similarity index 100% rename from files/collect-container-logs.sh rename to roles/collect_logs/files/collect-container-logs.sh diff --git a/files/heat-deploy-times.py b/roles/collect_logs/files/heat-deploy-times.py similarity index 100% rename from files/heat-deploy-times.py rename to roles/collect_logs/files/heat-deploy-times.py diff --git a/roles/collect_logs/library b/roles/collect_logs/library new file mode 120000 index 0000000..eab9f86 --- /dev/null +++ b/roles/collect_logs/library @@ -0,0 +1 @@ +../../plugins/modules \ No newline at end of file diff --git a/meta/main.yml b/roles/collect_logs/meta/main.yml similarity index 100% rename from meta/main.yml rename to roles/collect_logs/meta/main.yml diff --git a/roles/collect_logs/module_utils/sova_lib.py b/roles/collect_logs/module_utils/sova_lib.py new file mode 100644 index 0000000..0d4f318 --- /dev/null +++ b/roles/collect_logs/module_utils/sova_lib.py @@ -0,0 +1,102 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gzip +import logging +import yaml + +try: + import regex as regex_module +except ImportError: + import re as regex_module + + +logging.basicConfig( + format=('%(asctime)s - %(name)s - %(levelname)s - ' + '%(module)s.%(funcName)s:%(lineno)d - %(message)s')) +log = logging.getLogger('parser') +log.setLevel(logging.ERROR) + + +class Pattern(object): + def __init__(self, data): + self.data = data + self.load_yaml() + self.setup_regexes() + self.setup_patterns() + + def load_yaml(self): + if isinstance(self.data, dict): + self.config = self.data + else: + self.config = yaml.safe_load(self.data) + + def setup_regexes(self): + self.regexes = {} + if self.config: + for regexp in self.config.get('regexes', []): + flags = [] + if regexp.get('multiline'): + flags.append(regex_module.MULTILINE) + self.regexes[regexp.get('name')] = regex_module.compile( + r'{}'.format(regexp.get('regex')), *flags) + + def setup_patterns(self): + self._patterns = self.config.get('patterns', {}) + if self._patterns: + for key in self._patterns: + for p in self._patterns[key]: + if p['pattern'] in self.regexes: + p['pattern'] = self.regexes[p['pattern']] + if p['logstash'] in self.regexes: + p['logstash'] = self.regexes[p['logstash']] + + @property + def patterns(self): + return self._patterns + + +def line_match(pat, line, exclude=None): + if isinstance(pat, str): + return pat in line + found = pat.search(line) + if not found: + return False + if found.groups(): + if exclude: + if any([i in found.group(1) for i in exclude]): + return False + return found.group(1) + return True + + +def parse(text_file, patterns): + ids = [] + msgs = [] + if text_file.split(".")[-1] == "gz": + open_func = gzip.open + else: + open_func = open + with open_func(text_file, "rt") as finput: + text = finput.read() + for p in patterns: + line_matched = line_match( + p["pattern"], text, exclude=p.get("exclude")) + if line_matched: + log.debug("Found pattern {} in file {}".format( + repr(p), text_file)) + ids.append(p["id"]) + msgs.append(p["msg"].format(line_matched)) + return list(set(ids)), list(set(msgs)) diff --git a/molecule/default/converge.yml b/roles/collect_logs/molecule/default/converge.yml similarity index 93% rename from molecule/default/converge.yml rename to roles/collect_logs/molecule/default/converge.yml index a450fdd..d0f9eac 100644 --- a/molecule/default/converge.yml +++ b/roles/collect_logs/molecule/default/converge.yml @@ -37,7 +37,7 @@ # brief call used a very short override artcl_commands, enough to validate # that the combining of the commands works. Later we import the role with # its default artcl_commands in order to test these commands, too. - - name: "Include ansible-role-collect-logs :: collect (brief)" + - name: "Include collect_logs :: collect (brief)" vars: artcl_collect: true artcl_commands: @@ -52,7 +52,7 @@ cmd: cat /proc/swaps capture_file: /var/log/extra/swaps.txt include_role: - name: "ansible-role-collect-logs" + name: collect_logs - name: Verify expected combined commands assert: @@ -73,7 +73,7 @@ vars: artcl_collect: true include_role: - name: "ansible-role-collect-logs" + name: collect_logs - name: "Converge publish play" hosts: localhost @@ -86,7 +86,7 @@ artcl_collect: false artcl_publish: true include_role: - name: "ansible-role-collect-logs" + name: collect_logs - debug: msg: | diff --git a/molecule/default/molecule.yml b/roles/collect_logs/molecule/default/molecule.yml similarity index 100% rename from molecule/default/molecule.yml rename to roles/collect_logs/molecule/default/molecule.yml diff --git a/molecule/default/prepare.yml b/roles/collect_logs/molecule/default/prepare.yml similarity index 100% rename from molecule/default/prepare.yml rename to roles/collect_logs/molecule/default/prepare.yml diff --git a/molecule/default/verify.yml b/roles/collect_logs/molecule/default/verify.yml similarity index 100% rename from molecule/default/verify.yml rename to roles/collect_logs/molecule/default/verify.yml diff --git a/molecule/infrared/converge.yml b/roles/collect_logs/molecule/infrared/converge.yml similarity index 98% rename from molecule/infrared/converge.yml rename to roles/collect_logs/molecule/infrared/converge.yml index f2f7c20..8b168c8 100644 --- a/molecule/infrared/converge.yml +++ b/roles/collect_logs/molecule/infrared/converge.yml @@ -33,7 +33,7 @@ - name: "Copy ansible-role-collect-logs to test host" synchronize: - src: "{{ playbook_dir }}/../../" + src: "{{ playbook_dir }}/../../../../" dest: "{{ ansible_env.HOME }}/artcl-src" rsync_opts: - "--exclude=.tox" diff --git a/molecule/infrared/molecule.yml b/roles/collect_logs/molecule/infrared/molecule.yml similarity index 100% rename from molecule/infrared/molecule.yml rename to roles/collect_logs/molecule/infrared/molecule.yml diff --git a/molecule/infrared/verify.yml b/roles/collect_logs/molecule/infrared/verify.yml similarity index 100% rename from molecule/infrared/verify.yml rename to roles/collect_logs/molecule/infrared/verify.yml diff --git a/molecule/sova/converge.yml b/roles/collect_logs/molecule/sova/converge.yml similarity index 61% rename from molecule/sova/converge.yml rename to roles/collect_logs/molecule/sova/converge.yml index 95fd4c4..205677b 100644 --- a/molecule/sova/converge.yml +++ b/roles/collect_logs/molecule/sova/converge.yml @@ -2,9 +2,9 @@ - name: Converge hosts: all tasks: - - name: "Include ansible-role-collect-logs" + - name: Include collect_logs include_role: - name: "ansible-role-collect-logs" + name: collect_logs tasks_from: sova.yml tags: - molecule-idempotence-notest diff --git a/molecule/sova/molecule.yml b/roles/collect_logs/molecule/sova/molecule.yml similarity index 100% rename from molecule/sova/molecule.yml rename to roles/collect_logs/molecule/sova/molecule.yml diff --git a/molecule/sova/prepare.yml b/roles/collect_logs/molecule/sova/prepare.yml similarity index 100% rename from molecule/sova/prepare.yml rename to roles/collect_logs/molecule/sova/prepare.yml diff --git a/molecule/sova/verify.yml b/roles/collect_logs/molecule/sova/verify.yml similarity index 100% rename from molecule/sova/verify.yml rename to roles/collect_logs/molecule/sova/verify.yml diff --git a/scripts/doc_extrapolation.awk b/roles/collect_logs/scripts/doc_extrapolation.awk similarity index 100% rename from scripts/doc_extrapolation.awk rename to roles/collect_logs/scripts/doc_extrapolation.awk diff --git a/tasks/collect.yml b/roles/collect_logs/tasks/collect.yml similarity index 100% rename from tasks/collect.yml rename to roles/collect_logs/tasks/collect.yml diff --git a/tasks/collect/container.yml b/roles/collect_logs/tasks/collect/container.yml similarity index 100% rename from tasks/collect/container.yml rename to roles/collect_logs/tasks/collect/container.yml diff --git a/tasks/collect/monitoring.yml b/roles/collect_logs/tasks/collect/monitoring.yml similarity index 100% rename from tasks/collect/monitoring.yml rename to roles/collect_logs/tasks/collect/monitoring.yml diff --git a/tasks/collect/network.yml b/roles/collect_logs/tasks/collect/network.yml similarity index 100% rename from tasks/collect/network.yml rename to roles/collect_logs/tasks/collect/network.yml diff --git a/tasks/collect/system.yml b/roles/collect_logs/tasks/collect/system.yml similarity index 100% rename from tasks/collect/system.yml rename to roles/collect_logs/tasks/collect/system.yml diff --git a/tasks/create-docs.yml b/roles/collect_logs/tasks/create-docs.yml similarity index 100% rename from tasks/create-docs.yml rename to roles/collect_logs/tasks/create-docs.yml diff --git a/tasks/main.yml b/roles/collect_logs/tasks/main.yml similarity index 100% rename from tasks/main.yml rename to roles/collect_logs/tasks/main.yml diff --git a/tasks/publish.yml b/roles/collect_logs/tasks/publish.yml similarity index 100% rename from tasks/publish.yml rename to roles/collect_logs/tasks/publish.yml diff --git a/tasks/publish_ara.yml b/roles/collect_logs/tasks/publish_ara.yml similarity index 100% rename from tasks/publish_ara.yml rename to roles/collect_logs/tasks/publish_ara.yml diff --git a/tasks/publish_ara_graphite.yml b/roles/collect_logs/tasks/publish_ara_graphite.yml similarity index 100% rename from tasks/publish_ara_graphite.yml rename to roles/collect_logs/tasks/publish_ara_graphite.yml diff --git a/tasks/publish_ara_influxdb.yml b/roles/collect_logs/tasks/publish_ara_influxdb.yml similarity index 100% rename from tasks/publish_ara_influxdb.yml rename to roles/collect_logs/tasks/publish_ara_influxdb.yml diff --git a/tasks/sanitize_log_strings.yaml b/roles/collect_logs/tasks/sanitize_log_strings.yaml similarity index 100% rename from tasks/sanitize_log_strings.yaml rename to roles/collect_logs/tasks/sanitize_log_strings.yaml diff --git a/tasks/sova.yml b/roles/collect_logs/tasks/sova.yml similarity index 100% rename from tasks/sova.yml rename to roles/collect_logs/tasks/sova.yml diff --git a/templates/full_logs.html.j2 b/roles/collect_logs/templates/full_logs.html.j2 similarity index 100% rename from templates/full_logs.html.j2 rename to roles/collect_logs/templates/full_logs.html.j2 diff --git a/templates/index.rst.j2 b/roles/collect_logs/templates/index.rst.j2 similarity index 100% rename from templates/index.rst.j2 rename to roles/collect_logs/templates/index.rst.j2 diff --git a/templates/odl_extra_logs.j2 b/roles/collect_logs/templates/odl_extra_logs.j2 similarity index 100% rename from templates/odl_extra_logs.j2 rename to roles/collect_logs/templates/odl_extra_logs.j2 diff --git a/templates/rsync-filter.j2 b/roles/collect_logs/templates/rsync-filter.j2 similarity index 100% rename from templates/rsync-filter.j2 rename to roles/collect_logs/templates/rsync-filter.j2 diff --git a/vars/family-redhat.yml b/roles/collect_logs/vars/family-redhat.yml similarity index 100% rename from vars/family-redhat.yml rename to roles/collect_logs/vars/family-redhat.yml diff --git a/vars/infrared-collect-exclude-list.yml b/roles/collect_logs/vars/infrared-collect-exclude-list.yml similarity index 100% rename from vars/infrared-collect-exclude-list.yml rename to roles/collect_logs/vars/infrared-collect-exclude-list.yml diff --git a/vars/sova-patterns.yml b/roles/collect_logs/vars/sova-patterns.yml similarity index 100% rename from vars/sova-patterns.yml rename to roles/collect_logs/vars/sova-patterns.yml diff --git a/vars/unsecure.yml b/roles/collect_logs/vars/unsecure.yml similarity index 100% rename from vars/unsecure.yml rename to roles/collect_logs/vars/unsecure.yml diff --git a/setup.cfg b/setup.cfg index 7bc042f..39caddd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,17 +19,11 @@ setup-hooks = pbr.hooks.setup_hook [files] +# Allows us to install the role using pip so Ansible can find it. data_files = - share/ansible/roles/collect-logs/defaults = defaults/* - share/ansible/roles/collect-logs/meta = meta/* - share/ansible/roles/collect-logs/tasks = tasks/* - share/ansible/roles/collect-logs/templates = templates/* - share/ansible/roles/collect-logs/files = files/* - share/ansible/roles/collect-logs/filter_plugins = filter_plugins/* - share/ansible/roles/collect-logs/library = library/* - share/ansible/roles/collect-logs/module_utils = module_utils/* - share/ansible/roles/collect-logs/vars = vars/* - share/ansible/roles/collect-logs/scripts = scripts/* + share/ansible/roles/collect-logs = roles/collect_logs/* + share/ansible/roles/collect-logs/library = plugins/modules/* + share/ansible/roles/collect-logs/module_utils = plugins/module_utils/* share/ansible/roles/collect-logs/docs = docs/* [wheel] diff --git a/test-playbooks/zuul-ansible-role-collect-logs.yaml b/test-playbooks/zuul-ansible-role-collect-logs.yaml index 25f3fe5..30c12cb 100644 --- a/test-playbooks/zuul-ansible-role-collect-logs.yaml +++ b/test-playbooks/zuul-ansible-role-collect-logs.yaml @@ -2,5 +2,8 @@ - hosts: all tasks: - name: include ansible-role-collect-logs role + vars: + artcl_collect: true + artcl_publish: true include_role: - name: collect-logs + name: collect_logs diff --git a/tests/test_flatten_nested_dict.py b/tests/test_flatten_nested_dict.py index f598412..3d23fc8 100644 --- a/tests/test_flatten_nested_dict.py +++ b/tests/test_flatten_nested_dict.py @@ -1,5 +1,6 @@ import pytest # noqa -import flatten_nested_dict +import os +import sys from common.utils import ( AnsibleExitJson, AnsibleFailJson, ModuleTestCase, set_module_args, ) @@ -22,6 +23,12 @@ data: group: system """ +# Temporary hack until we adopt official ansible-test unit-testing +dir = os.path.join(os.path.dirname(__file__), "../roles/collect_logs/library") +sys.path.append(dir) +print(dir) +import flatten_nested_dict # noqa: E402 + class TestFlattenNestedDict(ModuleTestCase): diff --git a/tox.ini b/tox.ini index 68b19e9..c05698e 100644 --- a/tox.ini +++ b/tox.ini @@ -3,6 +3,8 @@ minversion = 3.4.0 envlist = docs, linters, molecule skipdist = True +requires = + tox-ansible >= 1.0.3 [testenv] usedevelop = True @@ -58,7 +60,7 @@ setenv = deps = ansible>=2.9,<2.10 molecule[test,docker]>=3.2.2,<3.3 # MIT - pytest-molecule + pytest-molecule>=1.3.4 pytest-plus # provides support for PYTEST_REQPASS commands = python -m pytest --color=yes --html={envlogdir}/reports.html --self-contained-html {tty:-s} {posargs} diff --git a/zuul.d/layout.yaml b/zuul.d/layout.yaml index e1e1033..413e0e0 100644 --- a/zuul.d/layout.yaml +++ b/zuul.d/layout.yaml @@ -1,4 +1,13 @@ --- +- job: + name: tox-ansible-test-sanity + description: Runs ansible-test sanity (tox -e sanity) + parent: tox + vars: + tox_envlist: sanity # dynamic tox env added by tox-ansible + # we want to run sanity only on py36 instead of implicit 2.6-3.9 range + tox_extra_args: -- --python 3.6 + - job: name: zuul-ansible-role-collect-logs description: Validate that zuul can use that role. @@ -6,7 +15,6 @@ run: test-playbooks/zuul-ansible-role-collect-logs.yaml roles: - zuul: opendev.org/openstack/ansible-role-collect-logs - name: collect-logs irrelevant-files: - ^vars/sova-patterns.yml$ @@ -19,6 +27,8 @@ jobs: &jobs - openstack-tox-linters - openstack-tox-molecule + - tox-ansible-test-sanity: + voting: false # until we fix reported errors # Limit the number of jobs executed while still assuring a relevant # level of coverage. If specific tasks are to be tested we should # consider implementing functional tests for them, especially as