linter: update ansible-lint; add auto-download of roles

This updates ansible-lint to the 6.x releases.  It also updates the
ansible installed to our current zuul version.

It cleans up the config file and marks it as yaml.  A few new
exceptions are added with explainations.

We don't need to fake the zuul_return by telling ansible-lint to mock
it.  This is added to the config, and we can remove the stub file.

A constant source of problems running this locally is that you have to
have the other role repos checked out so ansible-lint can do its
checks.  Add a smaller helper script for doing this locally.  In the
gate, ANSIBLE_ROLES_PATH is set to the Zuul checkouts of these
projects.  Locally, add a smaller helper "ansible-lint-roles-cache.sh"
that pulls the projects into a .cache directory.  If they are already
there, they get updated.  By default locally we will use these
checkouts.  This way, "tox -e linters" just works without having to do
anything else.

This also modifies the xargs to run the check all at once, instead of
fork for each file.  I did try autodetection but it seems like other
yaml files in the roles/playbook directories still confuse
ansible-lint.

Also I don't think we need a ansible-playbook --syntax-check step;
ansible-lint covers that.

Change-Id: I972f73037b9f904a555b81f3835ca5261639ed01

.ansible-lint

@@ -1,18 +1,30 @@
 exclude_paths:
-    - zuul.d/
-    - grafana/
-    - nodepool/
-    - ../../opendev
-    - ../../zuul
+  # don't lint external projects here
+  - ../../opendev
+  - ../../zuul
+  - ../openstack-zuul-jobs
+  - .cache/
+  - zuul.d/
+  - grafana/
+  - nodepool/
 parseable: true
 quiet: false
 skip_list:
-    - '106'  # Role name does not match ``^[a-z][a-z0-9_]+$`` pattern
-    - '204'  # [E204] Lines should be no longer than 120 chars
-             # TODO(AJaeger): E208 gives false positives currently, we
-             # should enable it once all issues in zuul-jobs and
-             # ansible-lint are fixed.
-    - '208'  # File permissions not mentioned
-    - '301'  # [E301] Commands should not change things if nothing needs doing
-    - '306'  # [E306] Shells that use pipes should set the pipefail option
+  - '106'  # Role name does not match ``^[a-z][a-z0-9_]+$`` pattern
+  - '204'  # [E204] Lines should be no longer than 120 chars
+           # TODO(AJaeger): E208 gives false positives currently, we
+           # should enable it once all issues in zuul-jobs and
+           # ansible-lint are fixed.
+  - '208'  # File permissions not mentioned
+  - '301'  # [E301] Commands should not change things if nothing needs doing
+  - '306'  # [E306] Shells that use pipes should set the pipefail option
+  - 'fqcn-builtins'  # for now ignore using fully-qualified names
+  - 'yaml[truthy]'   # doesn't like "yes"?
+  - 'yaml[line-length]'
 verbosity: 1
+mock_modules:
+  - zuul_return
+
+# Local variables:
+# mode: yaml
+# End:

tests/fake-ansible/library/zuul_return.py

@@ -1,12 +0,0 @@
-# This is a fake zuul_return to make ansible-lint happy
-from ansible.module_utils.basic import AnsibleModule
-
-
-def main():
-    return AnsibleModule(
-        argument_spec=dict(
-            data=dict(default=None),
-            path=dict(default=None, type=str),
-            file=dict(default=None, type=str),
-        )
-    )

tools/ansible-lint-roles-cache.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# This is a bit lame, but if we're running under Zuul then this is set
+# to the zuul checkouts of the required roles, so no need to do
+# anything here.
+if [[ ! ${ANSIBLE_ROLES_PATH} =~ \.cache.* ]]; then
+    exit 0
+fi
+
+if [ ! -d .cache/ansible-lint ]; then
+    mkdir -p .cache/ansible-lint
+fi
+
+pushd .cache/ansible-lint
+
+repos=(opendev/base-jobs
+        opendev/system-config
+        openstack/openstack-zuul-jobs
+        zuul/zuul-jobs)
+
+for repo in ${repos[@]}; do
+    dir=$(dirname $repo)
+    echo "Updating Ansible roles repo ${dir}"
+    if [ ! -d $repo ]; then
+        echo "Cloning fresh"
+        mkdir -p $dir
+        pushd $dir
+        git clone https://opendev.org/$repo
+        popd
+    else
+        echo "Updating repo"
+        pushd $repo
+        git fetch -a
+        git pull
+        popd
+    fi
+    echo "Done"
+done

tools/run-bashate.sh

@@ -6,5 +6,5 @@
 
 ROOT=$(readlink -fn $(dirname $0)/.. )
 find $ROOT -not -path '*playbooks/legacy/*' -and -not -wholename \*.tox/\* \
-    -and -not -wholename \*.test/\* \
+    -and -not -wholename \*.test/\* -and -not -wholename \*.cache/\* \
     -and -name \*.sh -print0 | xargs -0 bashate --ignore E006,E011

tox.ini

@@ -10,18 +10,11 @@ basepython = python3
 [testenv:linters]
 whitelist_externals = bash
 setenv =
-  ANSIBLE_LIBRARY= {toxinidir}/tests/fake-ansible
-  # NOTE(pabelanger): if you'd like to run tox -elinters locally,
-  # you'll need to export ANSIBLE_ROLES_PATH pointing to the required
-  # repos.
-  #
-  # We take a guess that zuul-jobs and openstack-zuul-jobs will be
-  # checked out in the directory above us (i.e. alongside
-  # project-config).  You may need to override this.  Make sure
-  # they're up to date too!
-  #
-  # see openstack-zuul-jobs-linters job for more information.
-  ANSIBLE_ROLES_PATH={env:ANSIBLE_ROLES_PATH:../zuul-jobs/roles:../openstack-zuul-jobs/roles:roles}
+  # NOTE(ianw) : In the gate job openstack-zuul-jobs-linters this is
+  # overridden to the Zuul checkouts of the projects.  Locally,
+  # tools/ansible-lint-roles-cache.sh will populate these for the
+  # linter.
+  ANSIBLE_ROLES_PATH={env:ANSIBLE_ROLES_PATH:{toxinidir}/.cache/ansible-lint/opendev/base-jobs/roles:{toxinidir}/.cache/ansible-lint/opendev/system-config/roles:{toxinidir}/.cache/ansible-lint/zuul/zuul-jobs/roles:{toxinidir}/.cache/ansible-lint/openstack/openstack-zuul-jobs/roles}
 
 # Add dependencies here since other jobs use python2 and zuul requires
 # python3.
@@ -30,24 +23,21 @@ deps =
      # keep this in sync with what Zuul uses so we don't allow though
      # things from a later ansible that would actually fail in
      # production.
-     ansible>=2.8,<=2.9
-     ansible-lint>=5.0.0,<6.0.0
+     ansible>=5,<6
+     ansible-lint>=6.3.0,<7.0.0
      hacking>=4.0.0,<5.0.0
      bashate>=0.2
 commands =
+  {toxinidir}/tools/ansible-lint-roles-cache.sh
   {toxinidir}/tools/run-bashate.sh
   flake8
   # Run some policy checks on zuul.d/projects.yaml
   {toxinidir}/tools/zuul-projects-checks.py
   # Ansible lint
   bash -c "find playbooks -type f -regex '.*.ya?ml' -print0 | \
-    xargs -t -n1 -0 ansible-lint"
-  bash -c 'find roles -maxdepth 1 -mindepth 1 -type d -printf "%p/\n" | \
-    xargs -t -n1 ansible-lint'
-  # Ansible Syntax Check
-  bash -c "find playbooks -type f -regex '.*.ya?ml' -exec \
-    ansible-playbook --syntax-check -i {toxinidir}/tests/inventory \
-    -e @tests/vars.yaml \{\} + > /dev/null"
+    xargs -t -0 ansible-lint"
+  bash -c "find roles -maxdepth 1 -mindepth 1 -type d -printf '%p/\n' | \
+    xargs -t ansible-lint"
   # Make sure site-variables at least parses
   python -c 'import yaml ; yaml.safe_load(open("zuul/site-variables.yaml", "r"))'