From 41153f065376336d560d64cfc21c1a2ce2ab484e Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Wed, 7 Dec 2022 12:05:07 -0800 Subject: [PATCH] Update zuul-jobs to handle tox3 and tox4 Tox 4 released today and is a complete rewrite with many backward incompatible changes. We need to update a number of things to support that in the zuul-jobs tox role and elsewhere. A possibly incomplete list of what was changed in this commit to make this work: * Don't run tox --showconfig with {{ tox_extra_args }} as -vv is in tox_extra_args by default and results in interleaved debug output in the ini output making it invalid ini content. * Update the tox siblings tox config parser to look for renamed environment dir locations. * Stop using whitelist_externals and use allowlist_exteranls because whitelist_externals is removed and external commands that are not explicitly allowed produce errors. * Make the tox version configurable in ensure-tox as some users may not be able to easily upgrade to tox v4. * Escape literal # chars in tox.ini as they are treated as comments when in the command strings now. https://github.com/tox-dev/tox/issues/2617 Change-Id: I38e13b4f13bb1b2d6fb7e5c70b708e9bb016a455 --- roles/ensure-tox/README.rst | 9 +++++++++ roles/ensure-tox/defaults/main.yaml | 1 + roles/ensure-tox/tasks/main.yaml | 2 +- .../library/tox_install_sibling_packages.py | 20 +++++++++++++++++-- roles/tox/tasks/siblings.yaml | 1 - test-playbooks/python/tox.ini | 4 +++- tox.ini | 8 +++++--- 7 files changed, 37 insertions(+), 8 deletions(-) diff --git a/roles/ensure-tox/README.rst b/roles/ensure-tox/README.rst index 26a79a94b..b949c90cf 100644 --- a/roles/ensure-tox/README.rst +++ b/roles/ensure-tox/README.rst @@ -5,6 +5,15 @@ virtual environment for the current user. **Role Variables** +.. zuul:rolevar:: ensure_tox_version + :default: '<4' + + Version specifier to select the version of tox. For example if your + project is not compatible with tox v4 you can set this value to + `<4` to install the latest v3 release. We have started this value + at `<4` for maximum compatibility, but expect it to change to '' + in the future when tox v4 is better understood. + .. zuul:rolevar:: tox_prefer_python2 :default: False diff --git a/roles/ensure-tox/defaults/main.yaml b/roles/ensure-tox/defaults/main.yaml index d210ecce0..e9830eddf 100644 --- a/roles/ensure-tox/defaults/main.yaml +++ b/roles/ensure-tox/defaults/main.yaml @@ -1,4 +1,5 @@ tox_executable: tox +ensure_tox_version: '<4' tox_venv_path: '{{ ansible_user_dir }}/.local/tox' tox_prefer_python2: false ensure_global_symlinks: false diff --git a/roles/ensure-tox/tasks/main.yaml b/roles/ensure-tox/tasks/main.yaml index e5ecae409..9e9c55da7 100644 --- a/roles/ensure-tox/tasks/main.yaml +++ b/roles/ensure-tox/tasks/main.yaml @@ -27,7 +27,7 @@ - name: Install tox to local venv # We pin tox to version <4 as v4 is not currently compatible with # zuul's tox siblings processing. - command: '{{ tox_venv_path }}/bin/pip install tox<4' + command: '{{ tox_venv_path }}/bin/pip install tox{{ ensure_tox_version }}' - name: Export installed tox_executable path set_fact: diff --git a/roles/tox/library/tox_install_sibling_packages.py b/roles/tox/library/tox_install_sibling_packages.py index 86ad5eec2..cbf81818c 100644 --- a/roles/tox/library/tox_install_sibling_packages.py +++ b/roles/tox/library/tox_install_sibling_packages.py @@ -272,6 +272,11 @@ def get_envlist(tox_config): # This is overly LBYL to deal with differences in older Python 2.7 # ConfigParser which would necessitate a fairly large number of exceptions # if we wanted to do a simple try/except with the get() instead + # Note this is tox<4 specific. tox>=4 is handled by the else block + # as tox>=4 does not provide tox.env (it is tox.env_list) and more + # importantly it does not provide args to check how we were called. + # But it does emit the appropriate testenv blocks depending on what -e + # value is used. if ( 'tox' in tox_config.sections() and 'env' in tox_config.options('tox') and "'-e" not in @@ -363,8 +368,19 @@ def main(): changed = False for testenv in envlist: - envdir = tox_config.get("testenv:{}".format(testenv), 'envdir') - envlogdir = tox_config.get("testenv:{}".format(testenv), 'envlogdir') + # Under tox<4 these names are envdir and envlogdir. Under tox>=4 + # they are env_dir and env_log_dir. + envname = "testenv:{}".format(testenv) + if tox_config.has_option(envname, 'envdir') and \ + tox_config.has_option(envname, 'envlogdir'): + envdir = tox_config.get(envname, 'envdir') + envlogdir = tox_config.get(envname, 'envlogdir') + elif tox_config.has_option(envname, 'env_dir') and \ + tox_config.has_option(envname, 'env_log_dir'): + envdir = tox_config.get(envname, 'env_dir') + envlogdir = tox_config.get(envname, 'env_log_dir') + else: + raise Exception("Unknown tox env directories") try: # Write a log file into the .tox dir so that it'll get picked up # Name it with testenv as a prefix so that fetch-tox-output diff --git a/roles/tox/tasks/siblings.yaml b/roles/tox/tasks/siblings.yaml index a25563420..1824dc02e 100644 --- a/roles/tox/tasks/siblings.yaml +++ b/roles/tox/tasks/siblings.yaml @@ -29,7 +29,6 @@ {% if tox_envlist is defined and tox_envlist %} -e{{ tox_envlist }} {% endif %} - {{ tox_extra_args }} > {{ _tox_show_config_tempfile.path }} args: chdir: "{{ zuul_work_dir }}" diff --git a/test-playbooks/python/tox.ini b/test-playbooks/python/tox.ini index a71264736..9ce66d8fc 100644 --- a/test-playbooks/python/tox.ini +++ b/test-playbooks/python/tox.ini @@ -3,7 +3,9 @@ minversion = 3.2 envlist = linters skipsdist = true # Test that we don't break when the requires feature is used -requires = tox-bindep +requires = + tox-bindep + tox<4 [testenv] whitelist_externals = sh diff --git a/tox.ini b/tox.ini index 3765b3195..f6ef3aa36 100644 --- a/tox.ini +++ b/tox.ini @@ -41,12 +41,14 @@ passenv = TERM setenv = GIT_PAGER: cat -whitelist_externals = bash +allowlist_externals = + bash + {toxinidir}/tools/check_jobs_documented.py + {toxinidir}/tools/update-test-platforms.py deps = -r{toxinidir}/linters-requirements.txt - commands = - bash -c "for f in $(find . -path './roles/*/library/*.py' | xargs); do echo Checking shebang on $f; if head -1 $f | grep -q '^#!'; then echo $f 'starts with #!, it should not as Ansible will choose the interpreter'; exit 1; fi; done" + bash -c "for f in $(find . -path './roles/*/library/*.py' | xargs); do echo Checking shebang on $f; if head -1 $f | grep -q '^\#!'; then echo $f 'starts with \#!, it should not as Ansible will choose the interpreter'; exit 1; fi; done" flake8 {posargs} yamllint -s -f parsable . python -m ansiblelint --version