Upgrade ansible-lint to 5.0

- bumps ansible-lint to 5.0
- updates our custom rules to make them compatible with 5.0
- replace custom module mocking with native ansible-lint ones
- remove custom call of ansible-playbook --syntax-check as now this
  is done by ansible-lint
- assured molecule vars are hosted under a vars/ folder in order to
  avoid confusing linter detection.
- replaced custom rule for loop var names in role as now this this an
  optional core feature of the linter (see config)
- replaced custom rule no-same-owner with opt-in one (see config)

Change-Id: I233fae8c9036d295968a97ee80e07fde8846c633
This commit is contained in:
Sorin Sbarnea 2021-01-08 08:57:26 +00:00
parent 78f18243f1
commit 0eaa5cf59a
54 changed files with 30 additions and 436 deletions
.ansible-lint
.rules
linters-requirements.txt
roles
ensure-bazelisk/tasks
ensure-package-repositories/tasks
ensure-rust/tasks
ensure-terraform/tasks
fetch-sphinx-output/tasks
fetch-tox-output/tasks
revoke-sudo/tasks
terraform/tasks
use-buildset-registry/tasks
test-playbooks
ansible-lint-rules
ZUULJOBS0001
faulty/roles
handlers-missing-loopvar-loop/handlers
handlers-missing-loopvar-with/handlers
task-block-missing-loopvar-loop/tasks
task-block-missing-loopvar-with/tasks
task-nested-block-missing-loopvar/tasks
tasks-include-missing-loopvar-loop/tasks
tasks-include-missing-loopvar-with/tasks
tasks-missing-loopvar-loop/tasks
tasks-missing-loopvar-with/tasks
valid/roles
handlers-loopvar-loop/handlers
handlers-loopvar-with/handlers
task-block-loopvar-loop/tasks
task-block-loopvar-with/tasks
task-missing-loopvar-loop-skip/tasks
tasks-include-loopvar-loop/tasks
tasks-include-loopvar-with/tasks
tasks-loopvar-loop/tasks
tasks-loopvar-with/tasks
ZUULJOBS0002
faulty/roles
block/tasks
nested-block/tasks
synchronize/tasks
unarchive-bz2/tasks
unarchive-delegated/tasks
unarchive-gz/tasks
unarchive-tar/tasks
unarchive-xz/tasks
unarchive-zip-same-owner/tasks
unarchive-zip/tasks
valid/roles
synchronize-delegated/tasks
synchronize-no-same-owner/tasks
unarchive-no-same-owner/tasks
unarchive-remote-src/tasks
unarchive-unknown-file-ending/tasks
ansible-lint.yamlrun.yaml
python
tests/fake-ansible/library
tox.ini
zuul-tests.d

@ -3,11 +3,16 @@ exclude_paths:
parseable: true parseable: true
quiet: false quiet: false
skip_list: skip_list:
- '106' # Role name does not match ``^[a-z][a-z0-9_]+$`` pattern - meta-no-info # No 'galaxy_info' found
- '204' # Lines should be no longer than 160 chars - no-changed-when # Commands should not change things if nothing needs doing
- '301' # Commands should not change things if nothing needs doing - no-tabs # Most files should not contain tabs
- '701' # No 'galaxy_info' found - role-name # Role name does not match ``^[a-z][a-z0-9_]+$`` pattern
rulesdir:
- ./.rules/
use_default_rules: true use_default_rules: true
verbosity: 1 verbosity: 1
mock_modules:
- zuul_console
- zuul_return
loop_var_prefix: zj_
# Enable rules that are disabled by default:
enable_list:
- no-same-owner

@ -1,55 +0,0 @@
from ansiblelint import AnsibleLintRule
class ZuulJobsNamespaceLoopVar(AnsibleLintRule):
id = 'ZUULJOBS0001'
shortdesc = 'Loop vars should have zj_ prefix'
description = """
Check for tasks that does not follow
the policy of namespacing loop variables with zj_ prefix.
See: \
https://zuul-ci.org/docs/zuul-jobs/policy.html\
#loops-in-roles
"""
tags = {'zuul-jobs-namespace-loop-var'}
def matchplay(self, file, task):
results = []
if file.get('type') not in ('tasks', 'handlers'):
return results
results.extend(self.handle_play(task))
return results
def handle_play(self, task):
results = []
if 'block' in task:
results.extend(self.handle_playlist(task['block']))
else:
results.extend(self.handle_task(task))
return results
def handle_playlist(self, playlist):
results = []
for play in playlist:
results.extend(self.handle_play(play))
return results
def handle_task(self, task):
results = []
has_loop = 'loop' in task
for key in task.keys():
if key.startswith('with_'):
has_loop = True
if has_loop:
if 'loop_control' not in task:
results.append(("", self.shortdesc))
elif 'loop_var' not in task.get('loop_control'):
results.append(("", self.shortdesc))
elif not task.get('loop_control')\
.get('loop_var').startswith('zj_'):
results.append(("", self.shortdesc))
return results

@ -1,81 +0,0 @@
import re
from ansiblelint import AnsibleLintRule
class ZuulJobsNoSameOwner(AnsibleLintRule):
id = 'ZUULJOBS0002'
shortdesc = 'Owner should not be kept between executor and remote'
description = """
Since there is no way to guarantee that the user and or group on the remote
node also exist on the executor and vice versa, owner and group should not
be preserved when transfering files between them.
See:
https://zuul-ci.org/docs/zuul-jobs/policy.html\
#preservation-of-owner-between-executor-and-remote
"""
tags = {'zuul-jobs-no-same-owner'}
def matchplay(self, file, play):
results = []
if file.get('type') not in ('tasks',
'handlers',
'playbooks'):
return results
results.extend(self.handle_play(play))
return results
def handle_play(self, task):
results = []
if 'block' in task:
results.extend(self.handle_playlist(task['block']))
else:
results.extend(self.handle_task(task))
return results
def handle_playlist(self, playlist):
results = []
for play in playlist:
results.extend(self.handle_play(play))
return results
def handle_task(self, task):
results = []
if 'synchronize' in task:
if self.handle_synchronize(task):
results.append(("", self.shortdesc))
elif 'unarchive' in task:
if self.handle_unarchive(task):
results.append(("", self.shortdesc))
return results
def handle_synchronize(self, task):
if task.get('delegate_to') is not None:
return False
synchronize = task['synchronize']
archive = synchronize.get('archive', True)
if synchronize.get('owner', archive) or\
synchronize.get('group', archive):
return True
return False
def handle_unarchive(self, task):
unarchive = task['unarchive']
delegate_to = task.get('delegate_to')
if delegate_to == 'localhost' or\
delegate_to != 'localhost' and 'remote_src' not in unarchive:
if unarchive['src'].endswith('zip'):
if '-X' in unarchive.get('extra_opts', []):
return True
if re.search(r'.*\.tar(\.(gz|bz2|xz))?$', unarchive['src']):
if '--no-same-owner' not in unarchive.get('extra_opts', []):
return True
return False

@ -1,5 +1,6 @@
# linters have different requirements than test ones, some would # linters have different requirements than test ones, some would
# conflict, like ansible version required by ansible-lint. # conflict, like ansible version required by ansible-lint.
ansible>=2.9,<2.11 # required by ansible-lint
flake8 flake8
yamllint>=1.23.0 yamllint>=1.23.0
ansible-lint>=4.3.7,<5 ansible-lint>=5.0.3rc1,<5.1

@ -19,7 +19,7 @@
register: bazelisk_downloaded register: bazelisk_downloaded
# This will apply to further plays and playbooks # This will apply to further plays and playbooks
- name: Set bazelisk_executable fact - name: Set bazelisk_executable fact # noqa no-handler
set_fact: set_fact:
bazelisk_executable: "{{ bazelisk_target }}" bazelisk_executable: "{{ bazelisk_target }}"
cacheable: true cacheable: true

@ -25,7 +25,7 @@
loop_var: zj_item loop_var: zj_item
register: _add_apt_repos register: _add_apt_repos
- name: Update APT cache - name: Update APT cache # noqa no-handler
become: true become: true
apt: apt:
update_cache: yes update_cache: yes

@ -1,5 +1,5 @@
- name: Install Rust - name: Install Rust
shell: | # noqa 303 shell: | # noqa command-instead-of-module
set -o pipefail set -o pipefail
curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain {{ ensure_rust_rustup_toolchain }} curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain {{ ensure_rust_rustup_toolchain }}
environment: environment:

@ -1,5 +1,5 @@
- name: Check if unzip is installed - name: Check if unzip is installed
command: unzip -v # noqa 303 command: unzip -v # noqa command-instead-of-module
failed_when: false failed_when: false
register: _unzip_probe register: _unzip_probe

@ -8,7 +8,7 @@
group: no group: no
when: not zuul_use_fetch_output when: not zuul_use_fetch_output
- name: Copy sphinx build html # noqa 208 - name: Copy sphinx build html # noqa risky-file-permissions
copy: copy:
dest: "{{ zuul_output_dir }}/logs/" dest: "{{ zuul_output_dir }}/logs/"
src: "{{ sphinx_output_src }}" src: "{{ sphinx_output_src }}"

@ -62,7 +62,7 @@
envlist: "{{ tox_all_environments.stdout_lines }}" envlist: "{{ tox_all_environments.stdout_lines }}"
when: tox_all_environments.stdout_lines is defined when: tox_all_environments.stdout_lines is defined
- name: Copy tox logs # noqa 208 - name: Copy tox logs # noqa risky-file-permissions
copy: copy:
dest: "{{ zuul_output_dir }}/logs/tox/" dest: "{{ zuul_output_dir }}/logs/tox/"
src: "{{ zuul_work_dir }}/.tox/{{ zj_testenv }}/log/" src: "{{ zuul_work_dir }}/.tox/{{ zj_testenv }}/log/"

@ -9,7 +9,7 @@
# try to delete the next file. # try to delete the next file.
- name: Remove sudo access for zuul user. - name: Remove sudo access for zuul user.
become: yes become: yes
command: rm -rf /etc/sudoers.d/zuul /etc/sudoers.d/90-cloud-init-users # noqa 302 command: rm -rf /etc/sudoers.d/zuul /etc/sudoers.d/90-cloud-init-users # noqa deprecated-command-syntax
when: zuul_is_sudoer.rc == 0 when: zuul_is_sudoer.rc == 0
- name: Prove that general sudo access is actually revoked. - name: Prove that general sudo access is actually revoked.

@ -89,7 +89,7 @@
- name: Get path to main.tf relative to the repo root - name: Get path to main.tf relative to the repo root
when: terraform_command == "plan" when: terraform_command == "plan"
register: main_file_location register: main_file_location
command: "git ls-files --full-name main.tf" # noqa 303 command: "git ls-files --full-name main.tf" # noqa command-instead-of-module
args: args:
chdir: "{{ zuul_work_dir }}" chdir: "{{ zuul_work_dir }}"

@ -42,7 +42,7 @@
dest: "{{ ca_dir }}/{{ buildset_registry_alias }}.crt" dest: "{{ ca_dir }}/{{ buildset_registry_alias }}.crt"
mode: 0644 mode: 0644
register: _tls_ca register: _tls_ca
- name: Update CA certs - name: Update CA certs # noqa: no-handler
command: "{{ ca_command }}" command: "{{ ca_command }}"
become: true become: true
when: _tls_ca is changed when: _tls_ca is changed

@ -1,5 +0,0 @@
- debug:
msg: "I should fail: {{ item }}"
loop:
- 1
- 2

@ -1,5 +0,0 @@
- debug:
msg: "I should fail: {{ item }}"
with_list:
- 1
- 2

@ -1,5 +0,0 @@
- debug:
msg: "I should error: {{ item }} "
loop:
- 1
- 2

@ -1,5 +0,0 @@
- debug:
msg: "I should error: {{ item }} "
with_list:
- 1
- 2

@ -1,8 +0,0 @@
- debug:
msg: "I should pass: {{ zj_item }}"
loop:
- 1
- 2
loop_control:
loop_var: zj_item

@ -1,8 +0,0 @@
- debug:
msg: "I should pass: {{ zj_item }}"
with_list:
- 1
- 2
loop_control:
loop_var: zj_item

@ -1,8 +0,0 @@
- block:
- debug:
msg: zj_item
loop:
- 1
- 2
loop_control:
loop_var: zj_item

@ -1,8 +0,0 @@
- block:
- debug:
msg: zj_item
with_items:
- 1
- 2
loop_control:
loop_var: zj_item

@ -1,5 +0,0 @@
- debug: # noqa ZUULJOBS0001
var: item
loop:
- 1
- 2

@ -1,7 +0,0 @@
- include: "{{ zj_item }}.yaml"
loop:
- 1
- 2
loop_control:
loop_var: zj_item

@ -1,6 +0,0 @@
- include: "{{ zj_item }}.yaml"
with_list:
- 1
- 2
loop_control:
loop_var: zj_item

@ -1,8 +0,0 @@
- debug:
msg: "I should pass: {{ zj_item }} "
loop:
- 1
- 2
loop_control:
loop_var: zj_item

@ -1,7 +0,0 @@
- debug:
msg: "I should pass: {{ zj_item }} "
with_list:
- 1
- 2
loop_control:
loop_var: zj_item

@ -1,4 +0,0 @@
- block:
- synchronize:
src: dummy
dest: dummy

@ -1,5 +0,0 @@
- block:
- block:
- synchronize:
src: dummy
dest: dummy

@ -1,3 +0,0 @@
- unarchive:
src: "{{ file }}.tar.bz2"
dest: "dummy"

@ -1,4 +0,0 @@
- unarchive:
src: "{{ file }}.tar.bz2"
dest: "dummy"
delegate_to: localhost

@ -1,3 +0,0 @@
- unarchive:
src: "{{ file }}.tar.gz"
dest: "dummy"

@ -1,3 +0,0 @@
- unarchive:
src: "{{ file }}.tar"
dest: "dummy"

@ -1,3 +0,0 @@
- unarchive:
src: "{{ file }}.tar.xz"
dest: "dummy"

@ -1,6 +0,0 @@
- unarchive:
src: "{{ file }}.zip"
dest: dummy
extra_opts:
- '-X'

@ -1,5 +0,0 @@
- unarchive:
src: "{{ file }}.zip"
dest: dummy
extra_opts:
- '-X'

@ -1,4 +0,0 @@
- synchronize:
src: dummy
dest: dummy
delegate_to: localhost

@ -1,5 +0,0 @@
- synchronize:
src: dummy
dest: dummy
owner: no
group: no

@ -1,5 +0,0 @@
- unarchive:
src: "{{ file }}.tar.gz"
dest: dummy
extra_opts:
- '--no-same-owner'

@ -1,4 +0,0 @@
- unarchive:
src: "{{ file }}.tar.xz"
dest: "dummy"
remote_src: true

@ -1,5 +0,0 @@
parseable: true
quiet: false
rulesdir:
- ../../.rules/
verbosity: 1

@ -1,61 +0,0 @@
- hosts: all
roles:
- ensure-pip
- ensure-virtualenv
tasks:
- name: Create tempdir for ansible-lint venv
tempfile:
state: directory
register: ansible_lint_tempdir
- name: Install linters-requirements.txt
pip:
requirements: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/linters-requirements.txt"
virtualenv: "{{ ansible_lint_tempdir.path }}"
- name: Make sure ansible-lint is installed
command: "{{ ansible_lint_tempdir.path }}/bin/ansible-lint --version"
- name: Get faulty playbooks and roles
command: >-
find test-playbooks/ansible-lint-rules/
-mindepth 4
-maxdepth 4
-wholename '*ZUULJOBS*/faulty/*/*'
args:
chdir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
register: faulty_ansible_items
- name: Get valid playbooks and roles
command: >-
find test-playbooks/ansible-lint-rules/
-mindepth 4
-maxdepth 4
-wholename '*ZUULJOBS*/valid/roles/*'
args:
chdir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
register: valid_ansible_items
- name: Make sure faulty roles fail linting
command: >-
{{ ansible_lint_tempdir.path }}/bin/ansible-lint
-c test-playbooks/ansible-lint-rules/ansible-lint.yaml
-t {{ item | regex_replace('.*/(ZUULJOBS.*?)/.*', '\1') }}
{{ item }}
args:
chdir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
register: ansible_lint
failed_when: ansible_lint.rc == 0
loop: "{{ faulty_ansible_items.stdout_lines }}"
- name: Make sure valid roles pass linting
command: >-
{{ ansible_lint_tempdir.path }}/bin/ansible-lint
-c test-playbooks/ansible-lint-rules/ansible-lint.yaml
-t {{ item | regex_replace('.*/(ZUULJOBS.*?)/.*', '\1') }}
{{ item }}
args:
chdir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
register: ansible_lint
failed_when: ansible_lint.rc != 0
loop: "{{ valid_ansible_items.stdout_lines }}"

@ -19,7 +19,6 @@
tox_envlist: docs,linters tox_envlist: docs,linters
tox_environment: tox_environment:
ANSIBLE_ROLES_PATH: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/roles" ANSIBLE_ROLES_PATH: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/roles"
ANSIBLE_LIBRARY: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/fake-ansible"
- name: Create tempfile to verify testenvs ran - name: Create tempfile to verify testenvs ran
tempfile: tempfile:

@ -1,6 +0,0 @@
# This is a fake zuul_return to make ansible-lint happy
from ansible.module_utils.basic import AnsibleModule
def main():
return AnsibleModule()

@ -1,12 +0,0 @@
# This is a fake zuul_return to make ansible-lint happy
from ansible.module_utils.basic import AnsibleModule
def main():
return AnsibleModule(
argument_spec=dict(
data=dict(default=None),
path=dict(default=None, type=str),
file=dict(default=None, type=str),
)
)

@ -38,8 +38,9 @@ passenv =
# to export ANSIBLE_ROLES_PATH pointing to the currect repos. # to export ANSIBLE_ROLES_PATH pointing to the currect repos.
# see openstack-zuul-jobs-linters job for more information. # see openstack-zuul-jobs-linters job for more information.
ANSIBLE_ROLES_PATH ANSIBLE_ROLES_PATH
TERM
setenv = setenv =
ANSIBLE_LIBRARY= {toxinidir}/tests/fake-ansible GIT_PAGER: cat
whitelist_externals = bash whitelist_externals = bash
deps = deps =
-r{toxinidir}/linters-requirements.txt -r{toxinidir}/linters-requirements.txt
@ -50,8 +51,6 @@ commands =
python -m ansiblelint --version python -m ansiblelint --version
python -m ansiblelint {env:ANSIBLELINT_OPTS:--progressive} python -m ansiblelint {env:ANSIBLELINT_OPTS:--progressive}
# Ansible Syntax Check # Ansible Syntax Check
bash -c "find playbooks -type f -regex '.*.ya?ml' ! -regex '.*vars\/.*' -exec \
ansible-playbook --syntax-check -i {toxinidir}/tests/inventory \{\} + > /dev/null"
{toxinidir}/tools/check_jobs_documented.py {toxinidir}/tools/check_jobs_documented.py
{toxinidir}/tools/update-test-platforms.py {toxinidir}/tools/update-test-platforms.py
bash -c "(( $(find playbooks -name *.yml | wc -l) == 0)) || \{ echo 'Use .yaml'; exit 1; \}" bash -c "(( $(find playbooks -name *.yml | wc -l) == 0)) || \{ echo 'Use .yaml'; exit 1; \}"

@ -1,18 +0,0 @@
- job:
name: zuul-jobs-test-ansible-lint-rules
description: |
Test custom ansible-lint rules in zuul-jobs
run: test-playbooks/ansible-lint-rules/run.yaml
files:
- ^\.rules/.*
- ^test-playbooks/ansible-lint-rules/.*
- ^\.ansible-lint
- project:
check:
jobs: &id001
- zuul-jobs-test-ansible-lint-rules
gate:
jobs: *id001
periodic-weekly:
jobs: *id001