From 4403a163d4b9aa252fd23bc0dfbf11ae296a503b Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Wed, 5 Jun 2019 08:24:43 -0500 Subject: [PATCH] Add update lint jobs using pre-commit Pre-commit was added to run all of the lint jobs via tox. Because this change now enforces the linters, the modules and playbooks have been updated so they're able to pass the checks. > Several excludes and lint ommisions have been added to the pre-commit config. These were added because the current code can not pass the basic checks in its present state. In a future PR updates will be made to the playbooks and modules and these ommissions will be removed. The goal of this PR is to ensure some checks are running and lay the ground work for future structural changes. Change-Id: If2095741dd0e977de71f8110307aef92d4f3676c Signed-off-by: Kevin Carter --- .pre-commit-config.yaml | 50 +++++++++++++ .yamllint | 7 ++ README.rst | 2 +- test-requirements.txt | 2 +- tox.ini | 75 ++++++++++++------- tripleo_ansible/ansible-role-requirements.yml | 21 ++++++ tripleo_ansible/container-cleanup.yml | 19 ++++- tripleo_ansible/fetch-logs.yml | 19 ++++- tripleo_ansible/library/podman_image.py | 9 ++- tripleo_ansible/restart-service.yml | 19 ++++- zuul.d/layout.yaml | 2 + 11 files changed, 184 insertions(+), 41 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 .yamllint create mode 100644 tripleo_ansible/ansible-role-requirements.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..4d64633ca --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,50 @@ +--- +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.1.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: mixed-line-ending + - id: check-byte-order-marker + - id: check-executables-have-shebangs + - id: check-merge-conflict + - id: debug-statements + - id: flake8 + entry: flake8 --ignore=E24,E121,E123,E124,E126,E226,E305,E402,F401,F405,E501,E704,F403,F841,W503 + # TODO(cloudnull): These codes were added to pass the lint check. + # All of these ignore codes should be resolved in + # future PRs. + - id: check-yaml + files: .*\.(yaml|yml)$ + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.15.0 + hooks: + - id: yamllint + files: \.(yaml|yml)$ + types: [file, yaml] + entry: yamllint --strict -f parsable + - repo: https://github.com/ansible/ansible-lint + rev: v4.1.0a0 + hooks: + - id: ansible-lint + files: \.(yaml|yml)$ + entry: >- + ansible-lint --force-color -v -x "ANSIBLE0006,ANSIBLE0007,ANSIBLE0010,ANSIBLE0012,ANSIBLE0013,ANSIBLE0016" + --exclude=tripleo_ansible/roles/openstack-operations/tasks/restore_galera.yml + --exclude=tripleo_ansible/roles/openstack-operations/tasks/restore_redis.yml + # TODO(cloudnull): These codes were added to pass the lint check. + # All of these ignore codes should be resolved in + # future PRs. + - repo: https://github.com/openstack-dev/bashate.git + rev: 0.6.0 + hooks: + - id: bashate + entry: bashate --error . --verbose --ignore=E006,E040 + # Run bashate check for all bash scripts + # Ignores the following rules: + # E006: Line longer than 79 columns (as many scripts use jinja + # templating, this is very difficult) + # E040: Syntax error determined using `bash -n` (as many scripts + # use jinja templating, this will often fail and the syntax + # error will be discovered in execution anyway) diff --git a/.yamllint b/.yamllint new file mode 100644 index 000000000..689b7b533 --- /dev/null +++ b/.yamllint @@ -0,0 +1,7 @@ +--- +extends: default + +rules: + line-length: + # matches hardcoded 160 value from ansible-lint + max: 160 diff --git a/README.rst b/README.rst index 3d3ec8813..1861eba96 100644 --- a/README.rst +++ b/README.rst @@ -6,7 +6,7 @@ TripleO Ansible project repository. Contains playbooks for use with TripleO Ope An inventory file can be obtained by running:: - tripleo-ansible-inventory + tripleo-ansible-inventory Operations Playbooks -------------------- diff --git a/test-requirements.txt b/test-requirements.txt index bab219ee0..0dbc7e8aa 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1 +1 @@ -hacking>=1.1.0,<1.2.0 # Apache-2.0 +pre-commit # MIT \ No newline at end of file diff --git a/tox.ini b/tox.ini index 939844bc4..624cd0397 100644 --- a/tox.ini +++ b/tox.ini @@ -6,12 +6,19 @@ skipdist = True [testenv] usedevelop = True install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt} {opts} {packages} -setenv = VIRTUAL_ENV={envdir} +passenv = * +setenv = + VIRTUAL_ENV={envdir} + ANSIBLE_ACTION_PLUGINS={toxinidir}/tripleo_ansible/ansible_plugins/action + ANSIBLE_CALLBACK_PLUGINS={toxinidir}/tripleo_ansible/ansible_plugins/callback + ANSIBLE_FILTER_PLUGINS={toxinidir}/tripleo_ansible/ansible_plugins/filter + ANSIBLE_LIBRARY={toxinidir}/tripleo_ansible/ansible_plugins/modules + ANSIBLE_MODULE_UTILS={toxinidir}/tripleo_ansible/ansible_plugins/module_utils + ANSIBLE_ROLES_PATH={toxinidir}/tripleo_ansible/roles deps = -r {toxinidir}/test-requirements.txt whitelist_externals = bash [testenv:bindep] -basepython = python3 # Do not install any requirements. We want this to be fast and work even if # system dependencies are missing, since it's used to tell you what system # dependencies are missing! This also means that bindep must be installed @@ -20,47 +27,57 @@ deps = bindep commands = bindep test [testenv:pep8] -basepython = python3 +envdir = {toxworkdir}/linters commands = - # Run hacking/flake8 check for all python files - bash -c "git ls-files | grep -v releasenotes | xargs grep --binary-files=without-match \ - --files-with-match '^.!.*python$' \ - --exclude-dir .tox \ - --exclude-dir .git \ - --exclude-dir .eggs \ - --exclude-dir *.egg-info \ - --exclude-dir dist \ - --exclude-dir *lib/python* \ - --exclude-dir doc \ - | xargs flake8 --verbose" + python -m pre_commit run flake8 -a [testenv:ansible-lint] -basepython=python2 -deps = -r {toxinidir}/ansible-requirements.txt +envdir = {toxworkdir}/linters +deps = {[testenv:linters]deps} commands = - bash ci-scripts/ansible-lint.sh + bash -c "ansible-galaxy install -fr {toxinidir}/tripleo_ansible/ansible-role-requirements.yml" + python -m pre_commit run ansible-lint -a + +[testenv:yamllint] +envdir = {toxworkdir}/linters +deps = {[testenv:linters]deps} +commands = + python -m pre_commit run yamllint -a + +[testenv:bashate] +envdir = {toxworkdir}/linters +deps = {[testenv:linters]deps} +commands = + python -m pre_commit run bashate -a + +[testenv:whitespace] +envdir = {toxworkdir}/linters +deps = {[testenv:linters]deps} +commands = + python -m pre_commit run trailing-whitespace -a + +[testenv:shebangs] +envdir = {toxworkdir}/linters +deps = {[testenv:linters]deps} +commands = + python -m pre_commit run check-executables-have-shebangs -a [testenv:linters] -basepython = python3 deps = -r {toxinidir}/test-requirements.txt - -r {toxinidir}/ansible-requirements.txt + -r {toxinidir}/requirements.txt commands = {[testenv:pep8]commands} {[testenv:ansible-lint]commands} + {[testenv:bashate]commands} + {[testenv:yamllint]commands} + {[testenv:whitespace]commands} + {[testenv:shebangs]commands} [testenv:releasenotes] -basepython = python3 whitelist_externals = bash -commands = bash -c ci-scripts/releasenotes_tox.sh +commands = + bash -c "ci-scripts/releasenotes_tox.sh" [testenv:venv] -basepython = python3 commands = {posargs} - -[flake8] -# E123, E125 skipped as they are invalid PEP-8. -# E265 deals with spaces inside of comments -show-source = True -ignore = E123,E125,E265 -builtins = _ diff --git a/tripleo_ansible/ansible-role-requirements.yml b/tripleo_ansible/ansible-role-requirements.yml new file mode 100644 index 000000000..e5de7ea8a --- /dev/null +++ b/tripleo_ansible/ansible-role-requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: openstack-operations + scm: git + src: https://github.com/openstack/ansible-role-openstack-operations + version: master + trackbranch: master diff --git a/tripleo_ansible/container-cleanup.yml b/tripleo_ansible/container-cleanup.yml index da720a893..e8f32cdd4 100644 --- a/tripleo_ansible/container-cleanup.yml +++ b/tripleo_ansible/container-cleanup.yml @@ -1,7 +1,22 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + - name: Cleanup container items hosts: "{{ target_hosts | default('all') }}" - become: yes - + become: true tasks: - name: Cleanup unused images, containers, and volumes import_role: diff --git a/tripleo_ansible/fetch-logs.yml b/tripleo_ansible/fetch-logs.yml index 180a1051e..a646b95cc 100644 --- a/tripleo_ansible/fetch-logs.yml +++ b/tripleo_ansible/fetch-logs.yml @@ -1,7 +1,22 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + - name: Fetch logs hosts: "{{ target_hosts | default('all') }}" - become: yes - + become: true tasks: - name: Fetch logs from remote systems import_role: diff --git a/tripleo_ansible/library/podman_image.py b/tripleo_ansible/library/podman_image.py index 6d5aa2547..baba5de57 100644 --- a/tripleo_ansible/library/podman_image.py +++ b/tripleo_ansible/library/podman_image.py @@ -399,10 +399,11 @@ class PodmanImageManager(object): split_on=' ', maxsplit=1): layer_ids = [] for line in lines.splitlines(): - if startswith and line.startswith(startswith) \ - or contains and contains in line: - splitline = line.rsplit(split_on, maxsplit) - layer_ids.append(splitline[1]) + _condition1 = (startswith and line.startswith(startswith)) + _condition2 = (contains and contains in line) + if _condition1 or _condition2: + splitline = line.rsplit(split_on, maxsplit) + layer_ids.append(splitline[1]) return(layer_ids[-1]) diff --git a/tripleo_ansible/restart-service.yml b/tripleo_ansible/restart-service.yml index 5f073745a..95f7bcb74 100644 --- a/tripleo_ansible/restart-service.yml +++ b/tripleo_ansible/restart-service.yml @@ -1,7 +1,22 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + - name: Restart OpenStack services hosts: "{{ target_hosts | default('all') }}" - become: yes - + become: true tasks: - name: Restart services import_role: diff --git a/zuul.d/layout.yaml b/zuul.d/layout.yaml index e1920044a..c9857acf4 100644 --- a/zuul.d/layout.yaml +++ b/zuul.d/layout.yaml @@ -1,3 +1,5 @@ +--- + - project: check: jobs: