Transform artcl into a collection

Because Ansible official testing tools (ansible-test) cannot be used
without a collection, we change the code layout to make conformant.

WARNING: The role is no longer considered to be named
"ansible-role-collect-logs" but "collect_logs" instead, with a
temporary alias called "collect-logs".

Checklist:
- [x] ansible-test sanity checks runs (does not need to pass)
- [x] zuul is still able to use the role
- [x] infrared is still able to use the role
- [x] molecule tests are running and passing
- [x] tripleo-ci jobs still collect the files

One symlink is still needed for infrared until related patch lands:
https://review.gerrithub.io/c/redhat-openstack/infrared/+/508861

Change-Id: Ib87622797a284d837ee579d9cccec0ed73306626
Story: TRIPLEOCI-305

MANIFEST.in

@@ -0,0 +1,5 @@
+global-exclude __pycache__
+exclude infrared_plugin
+exclude zuul.d
+exclude test-playbooks
+exclude plugins

README.rst

@@ -1,4 +1,4 @@
-collect-logs
+collect_logs
 ============
 
 Ansible role for aggregating logs from different nodes.
@@ -221,7 +221,7 @@ Example Role Playbook
    - name: Gather logs
      hosts: all:!localhost
      roles:
-       - collect-logs
+       - collect_logs
 
 ** Note:
   The tasks that collect data from the nodes are executed with ignore_errors.

ansible.cfg

@@ -6,3 +6,8 @@ callback_whitelist = profile_tasks
 
 # Attempt to load custom modules whether it's installed system-wide or from a virtual environment
 roles_path = roles:$VIRTUAL_ENV/share/ansible/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:$VIRTUAL_ENV/usr/share/ansible/roles
+
+# Required by infrared
+host_key_checking = False
+forks = 500
+timeout = 300

galaxy.yml

@@ -0,0 +1,25 @@
+---
+name: collect_logs
+namespace: tripleo
+version: 0.0.1
+authors:
+  - tripleo
+readme: README.rst
+
+build_ignore:
+  - .ansible
+  - .cache
+  - .github
+  - .gitignore
+  - .pytest_cache
+  - .vscode
+  - .tox
+  - dist
+  - tox.ini
+  - .eggs
+  - .mypy_cache
+  - "*.egg-info"
+  - modules
+  - module_utils
+  - infrared_plugin
+  - scripts

infrared_plugin/main.yml

@@ -24,7 +24,7 @@
 
     - name: Ansible role collect logs
       include_role:
-        name: ansible-role-collect-logs
+        name: collect_logs
 
 # This section takes care of preparing the collected data for publishing
 # and for publishing itself
@@ -65,7 +65,7 @@
 
     - name: Ansible role collect logs
       include_role:
-        name: ansible-role-collect-logs
+        name: collect_logs
       when: artcl_publish|default(false)|bool
 
     - name: Delete artifact files from localhost

infrared_plugin/plugin.spec

@@ -3,7 +3,7 @@
 config:
   plugin_type: other
   entry_point: main.yml
-  roles_path: ../
+  roles_path: ../roles/
 subparsers:
   ansible-role-collect-logs:
     description: An Ansible role for aggregating logs from different nodes.

infrared_plugin/roles

@@ -0,0 +1 @@
+../roles

requirements.txt

@@ -1,2 +1,3 @@
 pbr>=1.6
-ansible>=2.5,<2.10
+# Do not remove 2.10, ansible-test and tox-ansible require it
+ansible>=2.5,<2.11

roles/collect_logs/library

@@ -0,0 +1 @@
+../../plugins/modules

roles/collect_logs/module_utils/sova_lib.py

@@ -0,0 +1,102 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import gzip
+import logging
+import yaml
+
+try:
+    import regex as regex_module
+except ImportError:
+    import re as regex_module
+
+
+logging.basicConfig(
+    format=('%(asctime)s - %(name)s - %(levelname)s - '
+            '%(module)s.%(funcName)s:%(lineno)d - %(message)s'))
+log = logging.getLogger('parser')
+log.setLevel(logging.ERROR)
+
+
+class Pattern(object):
+    def __init__(self, data):
+        self.data = data
+        self.load_yaml()
+        self.setup_regexes()
+        self.setup_patterns()
+
+    def load_yaml(self):
+        if isinstance(self.data, dict):
+            self.config = self.data
+        else:
+            self.config = yaml.safe_load(self.data)
+
+    def setup_regexes(self):
+        self.regexes = {}
+        if self.config:
+            for regexp in self.config.get('regexes', []):
+                flags = []
+                if regexp.get('multiline'):
+                    flags.append(regex_module.MULTILINE)
+                self.regexes[regexp.get('name')] = regex_module.compile(
+                    r'{}'.format(regexp.get('regex')), *flags)
+
+    def setup_patterns(self):
+        self._patterns = self.config.get('patterns', {})
+        if self._patterns:
+            for key in self._patterns:
+                for p in self._patterns[key]:
+                    if p['pattern'] in self.regexes:
+                        p['pattern'] = self.regexes[p['pattern']]
+                    if p['logstash'] in self.regexes:
+                        p['logstash'] = self.regexes[p['logstash']]
+
+    @property
+    def patterns(self):
+        return self._patterns
+
+
+def line_match(pat, line, exclude=None):
+    if isinstance(pat, str):
+        return pat in line
+    found = pat.search(line)
+    if not found:
+        return False
+    if found.groups():
+        if exclude:
+            if any([i in found.group(1) for i in exclude]):
+                return False
+        return found.group(1)
+    return True
+
+
+def parse(text_file, patterns):
+    ids = []
+    msgs = []
+    if text_file.split(".")[-1] == "gz":
+        open_func = gzip.open
+    else:
+        open_func = open
+    with open_func(text_file, "rt") as finput:
+        text = finput.read()
+        for p in patterns:
+            line_matched = line_match(
+                p["pattern"], text, exclude=p.get("exclude"))
+            if line_matched:
+                log.debug("Found pattern {} in file {}".format(
+                    repr(p), text_file))
+                ids.append(p["id"])
+                msgs.append(p["msg"].format(line_matched))
+    return list(set(ids)), list(set(msgs))

roles/collect_logs/molecule/default/converge.yml

@@ -37,7 +37,7 @@
     # brief call used a very short override artcl_commands, enough to validate
     # that the combining of the commands works. Later we import the role with
     # its default artcl_commands in order to test these commands, too.
-    - name: "Include ansible-role-collect-logs :: collect (brief)"
+    - name: "Include collect_logs :: collect (brief)"
       vars:
         artcl_collect: true
         artcl_commands:
@@ -52,7 +52,7 @@
               cmd: cat /proc/swaps
               capture_file: /var/log/extra/swaps.txt
       include_role:
-        name: "ansible-role-collect-logs"
+        name: collect_logs
 
     - name: Verify expected combined commands
       assert:
@@ -73,7 +73,7 @@
       vars:
         artcl_collect: true
       include_role:
-        name: "ansible-role-collect-logs"
+        name: collect_logs
 
 - name: "Converge publish play"
   hosts: localhost
@@ -86,7 +86,7 @@
         artcl_collect: false
         artcl_publish: true
       include_role:
-        name: "ansible-role-collect-logs"
+        name: collect_logs
 
     - debug:
         msg: |

roles/collect_logs/molecule/infrared/converge.yml

@@ -33,7 +33,7 @@
 
     - name: "Copy ansible-role-collect-logs to test host"
       synchronize:
-        src: "{{ playbook_dir }}/../../"
+        src: "{{ playbook_dir }}/../../../../"
         dest: "{{ ansible_env.HOME }}/artcl-src"
         rsync_opts:
           - "--exclude=.tox"

roles/collect_logs/molecule/sova/converge.yml

@@ -2,9 +2,9 @@
 - name: Converge
   hosts: all
   tasks:
-    - name: "Include ansible-role-collect-logs"
+    - name: Include collect_logs
       include_role:
-        name: "ansible-role-collect-logs"
+        name: collect_logs
         tasks_from: sova.yml
       tags:
         - molecule-idempotence-notest

setup.cfg

@@ -19,17 +19,11 @@ setup-hooks =
     pbr.hooks.setup_hook
 
 [files]
+# Allows us to install the role using pip so Ansible can find it.
 data_files =
-    share/ansible/roles/collect-logs/defaults = defaults/*
-    share/ansible/roles/collect-logs/meta = meta/*
-    share/ansible/roles/collect-logs/tasks = tasks/*
-    share/ansible/roles/collect-logs/templates = templates/*
-    share/ansible/roles/collect-logs/files = files/*
-    share/ansible/roles/collect-logs/filter_plugins = filter_plugins/*
-    share/ansible/roles/collect-logs/library = library/*
-    share/ansible/roles/collect-logs/module_utils = module_utils/*
-    share/ansible/roles/collect-logs/vars = vars/*
-    share/ansible/roles/collect-logs/scripts = scripts/*
+    share/ansible/roles/collect-logs = roles/collect_logs/*
+    share/ansible/roles/collect-logs/library = plugins/modules/*
+    share/ansible/roles/collect-logs/module_utils = plugins/module_utils/*
     share/ansible/roles/collect-logs/docs = docs/*
 
 [wheel]

test-playbooks/zuul-ansible-role-collect-logs.yaml

@@ -2,5 +2,8 @@
 - hosts: all
   tasks:
     - name: include ansible-role-collect-logs role
+      vars:
+        artcl_collect: true
+        artcl_publish: true
       include_role:
-        name: collect-logs
+        name: collect_logs

tests/test_flatten_nested_dict.py

@@ -1,5 +1,6 @@
 import pytest  # noqa
-import flatten_nested_dict
+import os
+import sys
 from common.utils import (
     AnsibleExitJson, AnsibleFailJson, ModuleTestCase, set_module_args,
 )
@@ -22,6 +23,12 @@ data:
     group: system
 """
 
+# Temporary hack until we adopt official ansible-test unit-testing
+dir = os.path.join(os.path.dirname(__file__), "../roles/collect_logs/library")
+sys.path.append(dir)
+print(dir)
+import flatten_nested_dict  # noqa: E402
+
 
 class TestFlattenNestedDict(ModuleTestCase):
 

tox.ini

@@ -3,6 +3,8 @@
 minversion = 3.4.0
 envlist = docs, linters, molecule
 skipdist = True
+requires =
+    tox-ansible >= 1.0.3
 
 [testenv]
 usedevelop = True
@@ -58,7 +60,7 @@ setenv =
 deps =
     ansible>=2.9,<2.10
     molecule[test,docker]>=3.2.2,<3.3  # MIT
-    pytest-molecule
+    pytest-molecule>=1.3.4
     pytest-plus  # provides support for PYTEST_REQPASS
 commands =
     python -m pytest --color=yes --html={envlogdir}/reports.html --self-contained-html {tty:-s} {posargs}

zuul.d/layout.yaml

@@ -1,4 +1,13 @@
 ---
+- job:
+    name: tox-ansible-test-sanity
+    description: Runs ansible-test sanity (tox -e sanity)
+    parent: tox
+    vars:
+      tox_envlist: sanity  # dynamic tox env added by tox-ansible
+      # we want to run sanity only on py36 instead of implicit 2.6-3.9 range
+      tox_extra_args: -- --python 3.6
+
 - job:
     name: zuul-ansible-role-collect-logs
     description: Validate that zuul can use that role.
@@ -6,7 +15,6 @@
     run: test-playbooks/zuul-ansible-role-collect-logs.yaml
     roles:
       - zuul: opendev.org/openstack/ansible-role-collect-logs
-        name: collect-logs
     irrelevant-files:
       - ^vars/sova-patterns.yml$
 
@@ -19,6 +27,8 @@
       jobs: &jobs
         - openstack-tox-linters
         - openstack-tox-molecule
+        - tox-ansible-test-sanity:
+            voting: false  # until we fix reported errors
         # Limit the number of jobs executed while still assuring a relevant
         # level of coverage. If specific tasks are to be tested we should
         # consider implementing functional tests for them, especially as