Upgrade ansible-lint/molecule runs

* bump linters
* sort newly identified rule violations
* fixes sanity by correcting missing excludes from galaxy.yml file
* fix incorrect pinning of python version in tox.ini

Change-Id: Iba0cecfc46a97d0ee56e30e48638f3a1b37bdec5

.gitignore

@@ -71,3 +71,6 @@ releasenotes/build
 # ansible retry files
 *.retry
 ansible_role_collect_logs.egg-info
+
+# buit collection
+*.tar.gz

.pre-commit-config.yaml

@@ -1,16 +1,16 @@
 ---
 repos:
   - repo: https://github.com/PyCQA/isort
-    rev: 5.8.0
+    rev: 5.9.3
     hooks:
       - id: isort
   - repo: https://github.com/python/black.git
-    rev: 20.8b1
+    rev: 21.7b0
     hooks:
       - id: black
         language_version: python3
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v4.0.1
     hooks:
       - id: end-of-file-fixer
       - id: trailing-whitespace
@@ -19,10 +19,8 @@ repos:
       - id: check-executables-have-shebangs
       - id: check-merge-conflict
       - id: debug-statements
-      - id: check-yaml
-        files: .*\.(yaml|yml)$
   - repo: https://gitlab.com/pycqa/flake8.git
-    rev: 3.9.1
+    rev: 3.9.2
     hooks:
       - id: flake8
         additional_dependencies:
@@ -30,7 +28,7 @@ repos:
           - flake8-black>=0.1.1
         language_version: python3
   - repo: https://github.com/ansible/ansible-lint.git
-    rev: v5.0.7
+    rev: v5.1.2
     hooks:
       - id: ansible-lint
         always_run: true
@@ -38,7 +36,9 @@ repos:
         # results when called with individual files.
         # https://github.com/ansible/ansible-lint/issues/611
         verbose: true
-        entry: env ANSIBLE_LIBRARY=./library ansible-lint --force-color -v .
+        additional_dependencies:
+          - ansible-core
+          - yamllint
   - repo: https://github.com/openstack-dev/bashate.git
     rev: 2.0.0
     hooks:

.yamllint

@@ -1,7 +0,0 @@
----
-extends: default
-
-rules:
-  line-length:
-    # matches hardcoded 160 value from ansible-lint
-    max: 160

MANIFEST.in

@@ -1,5 +1,11 @@
 global-exclude __pycache__
+exclude .benchmarks
+exclude .eggs
+exclude .mypy_cache
+exclude .pytest_cache
+exclude .quickstart
+exclude .tox
 exclude infrared_plugin
-exclude zuul.d
-exclude test-playbooks
 exclude plugins
+exclude test-playbooks
+exclude zuul.d

galaxy.yml

@@ -7,19 +7,39 @@ authors:
 readme: README.rst
 
 build_ignore:
+  - "**/.mypy_cache"
+  - "**/.pytest_cache"
+  - "*.egg-info"
+  - .DS_Store
   - .ansible
+  - .benchmarks
   - .cache
+  - .eggs
+  - .envrc
   - .github
   - .gitignore
-  - .pytest_cache
-  - .vscode
-  - .tox
-  - dist
-  - tox.ini
-  - .eggs
+  - .gitreview
   - .mypy_cache
-  - "*.egg-info"
-  - modules
-  - module_utils
+  - .pre-commit-config.yaml
+  - .pytest_cache
+  - .quickstart
+  - .tox
+  - .vscode
+  - .yamllint
+  - ansible.cfg
+  - bindep.txt
+  - build
+  - dist
+  - docs/source/_build
   - infrared_plugin
+  - module_utils
+  - modules
+  - pyproject.toml
+  - report.html
+  - roles/collect_logs/library
   - scripts
+  - setup.cfg
+  - setup.py
+  - test-playbooks
+  - tox.ini
+  - zuul.d

requirements.yml

@@ -0,0 +1,2 @@
+collections:
+  - name: ansible.posix  # needed by synchronize

roles/collect_logs/files/collect-container-logs.sh

@@ -76,12 +76,12 @@ for cont in $(${engine} ps | awk '{print $NF}' | grep -v NAMES); do
     # Capture rpms updated from more recent repos
     update_repos="gating delorean-current"
     if ls /etc/yum.repos.d/*-component.repo 1> /dev/null 2>&1; then
-        component_name=$(cat $(ls /etc/yum.repos.d/*-component.repo) | grep "name=" | sed "s/name=//g")
+        component_name=$(cat /etc/yum.repos.d/*-component.repo | grep "name=" | sed "s/name=//g")
         update_repos="${update_repos} ${component_name}"
     fi
     echo "*** ${cont} rpm update info ***" >> "$BASE_CONTAINER_EXTRA/container_updates_info.log"
     for repo in $update_repos; do
-        cat "$INFO_DIR/${engine}_info.log" | grep "@${repo}" >> "$BASE_CONTAINER_EXTRA/container_updates_info.log"
+        grep "@${repo}" "$INFO_DIR/${engine}_info.log" >> "$BASE_CONTAINER_EXTRA/container_updates_info.log"
     done;
 
     # NOTE(flaper87): This should go away. Services should be

roles/collect_logs/molecule/default/converge.yml

@@ -8,6 +8,7 @@
       file:
         path: "{{ local_working_dir }}"
         state: directory
+        mode: 0755
 
     # Assure we do not have leftovers from previous runs there, not needed in production.
     - name: Remove collected_files folder
@@ -88,6 +89,7 @@
       include_role:
         name: collect_logs
 
-    - debug:
+    - name: Display stats
+      debug:
         msg: |
           Collected files should be under {{ local_working_dir }}/collected_files

roles/collect_logs/molecule/default/prepare.yml

@@ -9,6 +9,7 @@
         content: |
           2000-00-00T00:00:00.000Z | this is sparta
         dest: /var/log/bootstrap-subnodes.log
+        mode: 0644
 
     - name: Remove /var/log/extra/logstash.txt
       file:

roles/collect_logs/molecule/default/verify.yml

@@ -4,5 +4,5 @@
   tasks:
 
     - name: Check logstash.txt contains expected data
-      shell: grep -q "this is sparta" /var/log/extra/logstash.txt
+      command: grep -q "this is sparta" /var/log/extra/logstash.txt
       changed_when: false

roles/collect_logs/molecule/infrared/converge.yml

@@ -30,6 +30,7 @@
       file:
         path: "{{ infrared_location }}/infrared_plugin"
         state: directory
+        mode: 0755
 
     - name: "Copy ansible-role-collect-logs to test host"
       synchronize:
@@ -56,6 +57,7 @@
       file:
         path: "{{ item }}"
         state: touch
+        mode: 0644
       with_items:
         - "/tmp/dummy.log"
         - "/tmp/append.log"

roles/collect_logs/molecule/infrared/molecule.yml

@@ -45,3 +45,5 @@ scenario:
     - verify
     - cleanup
     - destroy
+markers:
+  - xfail  # https://projects.engineering.redhat.com/browse/RHOSINFRA-4174

roles/collect_logs/molecule/infrared/verify.yml

@@ -24,7 +24,7 @@
 
     - name: Get directory with collected log files
       stat:
-        path: "{{ infrared_location}}/collected_files_test"
+        path: "{{ infrared_location }}/collected_files_test"
       register:
         collected_files_dir
 

roles/collect_logs/molecule/sova/prepare.yml

@@ -7,9 +7,11 @@
       file:
         path: '{{ ansible_user_dir }}/workspace/logs/'
         state: directory
+        mode: 0755
 
     - name: Create a sample console file
       copy:
         content: |
           No valid host was found. There are not enough hosts
         dest: '{{ ansible_user_dir }}/workspace/logs/quickstart_install.log'
+        mode: 0644

roles/collect_logs/tasks/collect.yml

@@ -11,11 +11,13 @@
       file:
         dest: /var/log/extra
         state: directory
+        mode: 0755
 
     - name: Create rsync filter file
       template:
         src: "odl_extra_logs.j2"
         dest: "/tmp/odl_extra_logs.sh"
+        mode: 0644
 
     - name: Determine commands to run
       run_once: true
@@ -90,11 +92,13 @@
   file:
     dest: "/tmp/{{ inventory_hostname }}"
     state: directory
+    mode: 0755
 
 - name: Create rsync filter file
   template:
     src: "rsync-filter.j2"
     dest: "/tmp/{{ inventory_hostname }}-rsync-filter"
+    mode: 0644
   when: artcl_rsync_collect_list|bool
 
 # This task needs to be finished before generating find list of files
@@ -166,7 +170,7 @@
   tags:
     - skip_ansible_lint
 
-- name: Create tar archive of logs for faster copying
+- name: Create tar archive of logs for faster copying  # noqa: command-instead-of-module
   shell:
     cmd: tar cf {{ inventory_hostname }}.tar {{ inventory_hostname }};
     chdir: /tmp
@@ -183,7 +187,7 @@
   file:
     path: "/tmp/{{ inventory_hostname }}"
     state: absent
-  ignore_errors: true
+  ignore_errors: true  # noqa ignore-errors
 
 - name: Extract the logs archive
   unarchive:

roles/collect_logs/tasks/collect/container.yml

@@ -12,12 +12,15 @@
       when: odl_container_enabled.rc != 0
 
 
-    - name: check if ODL is enabled via rpm
+    - name: check if ODL is enabled via rpm  # noqa: command-instead-of-module
       shell: rpm -qa | grep opendaylight
       register: odl_rpm_enabled
 
     - name: Create ODL log directory
-      file: dest="{{ odl_extra_log_dir }}" state=directory
+      file:
+        dest: "{{ odl_extra_log_dir }}"
+        state: directory
+        mode: 0755
       when: (odl_rpm_enabled.rc == 0) or (odl_container_enabled.rc == 0)
 
     - name: Collect OVS outputs for ODL
@@ -34,6 +37,7 @@
       copy:
         src: collect-container-logs.sh
         dest: /tmp/collect-container-logs.sh
+        mode: 0755
 
     - name: Run container logs collection with timeout
       command: >-

roles/collect_logs/tasks/create-docs.yml

@@ -11,14 +11,14 @@
        "{{ artcl_collect_dir }}/undercloud/home/{{ undercloud_user }}/{{ item }}.sh" > \
        "{{ artcl_docs_source_dir }}/{{ item }}.rst"
   with_items: "{{ artcl_create_docs_payload.included_deployment_scripts }}"
-  ignore_errors: true
+  ignore_errors: true  # noqa: ignore-errors
   changed_when: true
 
 - name: Fetch static rST docs to include in output docs
   shell: >
     cp "{{ artcl_docs_source_dir }}/../static/{{ item }}.rst" "{{ artcl_docs_source_dir }}"
   with_items: "{{ artcl_create_docs_payload.included_static_docs }}"
-  ignore_errors: true
+  ignore_errors: true  # noqa: ignore-errors
   changed_when: true
 
 - name: Generate fresh index.rst for Sphinx
@@ -26,11 +26,13 @@
     src: index.rst.j2
     dest: "{{ artcl_docs_source_dir }}/index.rst"
     force: true
+    mode: 0644
 
 - name: Ensure docs dir exists
   file:
     path: "{{ artcl_collect_dir }}/docs"
     state: directory
+    mode: 0755
 
 - name: Build docs with Sphinx
   shell: >
@@ -39,5 +41,5 @@
     2>&1 {{ timestamper_cmd }} > {{ artcl_collect_dir }}/docs/sphinx_build.log
   args:
     executable: /bin/bash
-  ignore_errors: true
+  ignore_errors: true  # noqa: ignore-errors
   changed_when: true

roles/collect_logs/tasks/publish.yml

@@ -61,7 +61,7 @@
   with_items:
     - "{{ local_working_dir }}/*.sh"
     - "{{ local_working_dir }}/*.log"
-  ignore_errors: true
+  ignore_errors: true  # noqa:  ignore-errors
   changed_when: true
 
 # the doc footer for logging has been removed.
@@ -71,7 +71,7 @@
     if [ -f {{ artcl_readme_path }} ]; then
         cp {{ artcl_readme_path }} {{ artcl_readme_file }};
     fi;
-  ignore_errors: true
+  ignore_errors: true  # noqa: ignore-errors
   changed_when: true
 
 - name: Rename compressed text based files to end with txt.gz extension
@@ -98,8 +98,8 @@
 
 # This is where upload starts, no local change after this line
 # All tags after this line are marked with "notest" to allow testing (molecule) in absence of access to log servers.
-- name: upload to the artifact server using pubkey auth
-  shell: >
+- name: upload to the artifact server using pubkey auth  # noqa: command-instead-of-module
+  command: >
     rsync -av
     --quiet -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {{ artcl_report_server_key|default('') }}"
     {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/{{ artcl_build_tag }}
@@ -110,10 +110,10 @@
   when: artcl_use_rsync|bool and not artcl_rsync_use_daemon|bool
   tags: notest
 
-- name: upload to the artifact server using password auth
+- name: upload to the artifact server using password auth  # noqa: command-instead-of-module
   environment:
     RSYNC_PASSWORD: "{{ artcl_rsync_password | default(omit) }}"
-  shell: rsync -av --quiet {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/{{ artcl_build_tag }}  # noqa 305
+  command: rsync -av --quiet {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/{{ artcl_build_tag }}
   async: "{{ artcl_publish_timeout }}"
   poll: 15
   retries: 5
@@ -149,5 +149,6 @@
   template:
     src: full_logs.html.j2
     dest: "{{ artcl_collect_dir }}/full_logs.html"
+    mode: 0644
   when: artcl_env != 'tripleo-ci'
   tags: notest

roles/collect_logs/tasks/publish_ara.yml

@@ -21,11 +21,11 @@
         {{ local_working_dir }}/bin/ara generate html {{ local_working_dir }}/ara_oooq_oc;
         {{ local_working_dir }}/bin/ara task list --all -f json > {{ artcl_collect_dir }}/ara.oooq.oc.json;
         cp -r {{ local_working_dir }}/ara_oooq_oc {{ artcl_collect_dir }}/;
-      ignore_errors: true
+      ignore_errors: true  # noqa: ignore-errors
       environment:
         ARA_DATABASE: 'sqlite:///{{ ara_overcloud_db_path }}'
 
-- name: Copy ara files to ara-report directories
+- name: Copy ara files to ara-report directories  # noqa: deprecated-command-syntax
   shell: |
     mkdir -p {{ artcl_collect_dir }}/{{ item.dir }}/ara-report;
     cp {{ item.file }} {{ artcl_collect_dir }}/{{ item.dir }}/ara-report/ansible.sqlite;

roles/collect_logs/tasks/publish_ara_influxdb.yml

@@ -17,6 +17,7 @@
     measurement: "{{ influxdb_measurement }}"
     data_file: "{{ influxdb_data_file_path }}"
     only_successful_tasks: "{{ influxdb_only_successful_tasks }}"
+  no_log: true
 
 - name: Get ARA json data for undercloud
   become: true
@@ -40,6 +41,7 @@
     standard_fields: false
     longest_tasks: 15
   when: ara_root_data.stdout != "[]"
+  no_log: true
 
 - name: Get ARA json data for overcloud
   shell: "{{ local_working_dir }}/bin/ara result list --all -f json"  # noqa 305
@@ -64,3 +66,4 @@
     standard_fields: false
     longest_tasks: 15
   when: ara_oc_data.stdout != "[]"
+  no_log: true

tests/sanity/ignore-2.10.txt

@@ -2,4 +2,3 @@ plugins/modules/ara_graphite.py validate-modules:missing-gplv3-license
 plugins/modules/ara_influxdb.py validate-modules:missing-gplv3-license
 plugins/modules/flatten_nested_dict.py validate-modules:missing-gplv3-license
 plugins/modules/sova.py validate-modules:missing-gplv3-license
-docs/static/env-setup-virt.rst rstcheck

tests/sanity/ignore-2.11.txt

@@ -0,0 +1 @@
+ignore-2.10.txt

tox.ini

@@ -4,7 +4,7 @@ minversion = 3.4.0
 envlist = docs, linters, molecule
 skipsdist = True
 requires =
-    tox-ansible >= 1.0.5
+    tox-ansible >= 1.5.3
 
 [testenv]
 usedevelop = True
@@ -56,17 +56,25 @@ setenv =
     ANSIBLE_SHOW_CUSTOM_STATS=1
     PYTHONPATH={env:PYTHONPATH:}:library
     # failsafe mechanism to avoid accidental disablement of tests
-    # 3 molecule tests expected
-    PYTEST_REQPASS=3
+    # 2/3 molecule scenarios are expected to pass
+    PYTEST_REQPASS=2
 deps =
     ansible>=2.9,<2.10
-    molecule[test,docker]>=3.2.2,<3.3  # MIT
-    pytest-molecule>=1.3.4
+    molecule[test,docker]>=3.3.2,<3.4  # MIT
+    pytest-molecule>=1.4.0
     pytest-plus  # provides support for PYTEST_REQPASS
 commands =
+    ansible-galaxy collection install community.docker
     python -m pytest --color=yes --html={envlogdir}/reports.html --self-contained-html {tty:-s} -k molecule {posargs}
 
-
 [testenv:ansible]
 description = Used as base for all tox-ansible environments
-basepython = python3.6
+
+# https://github.com/ansible-community/tox-ansible/issues/96
+[testenv:sanity]
+usedevelop = False
+deps =
+
+[testenv:units]
+usedevelop = False
+deps =