From 67f223b53a0d714f3220a2880e72b1fce7ad395f Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Thu, 18 Jun 2020 09:24:50 +1000 Subject: [PATCH] Partial revert "Ensure wheel exists for build-release-python"; move to ensure-pip This partially reverts commit 3f961ce202d7d24e2944de09636b35cec9c13bf6. This alternative installs wheel with the ensure-pip role instead of in a separate role. wheel is very closely linked with pip install operations so this isn't a large overreach of the role. I suggest this for several reasons; firstly the python-wheel role doesn't try to install packages, so we end up with mixed system pip and upstream versions of wheel most of the time. This is the type of thing that has proven problematic in the past. It also installs via pip --user; something we've already had problems with tox when for various reasons roles want to run this as non-zuul user. Using ensure-pip we keep the packaged versions together. [1] did try to install wheel with root, but during runtime which didn't work due to sudo being revoked. This should work for the existing build-python-release job, because it already includes ensure-pip in pre-run via playbooks/python/pre.yaml I believe our conclusion on the ensure-* roles was that requiring root/become: for installation is OK, but we should have a no-op path if the tools are found. This is consistent with that approach (i.e. if you want wheel and can't do sudo, you should pre-install it on your image using whatever you build that with). This adds a check to the existing "is pip installed" check to also check if wheel packages are available. If not we trigger the install path. This revealed some issues with RedHat.yaml -- we can always install Python 3 (packages available for CentOS 7) so remove that check, and if Ansible is running under Python 2; ensure we install the dependencies too (not only if it is forced). Update the documentation to describe that it will enable support for bdist_wheel, and add a basic sanity test that wheels are produced by pip. The existing build-python-release job is kept; although it is modified to use the playbooks/python/pre.yaml playbook as the build job does. Change-Id: I2ab11bb45b6b2a49d54db39195228ab40141185c [1] https://review.opendev.org/#/c/736001/5/roles/build-python-release/tasks/main.yaml --- doc/source/python-roles.rst | 1 - playbooks/python/release-pre.yaml | 4 - roles/build-python-release/README.rst | 3 +- roles/ensure-pip/README.rst | 3 + roles/ensure-pip/tasks/Debian.yaml | 2 + roles/ensure-pip/tasks/Gentoo.yaml | 8 +- roles/ensure-pip/tasks/RedHat.yaml | 7 +- roles/ensure-pip/tasks/Suse.yaml | 8 +- roles/ensure-pip/tasks/main.yaml | 2 + roles/ensure-pip/tasks/source.yaml | 7 ++ roles/ensure-pip/tasks/xenial.yaml | 2 + roles/ensure-wheel/README.rst | 9 -- roles/ensure-wheel/defaults/main.yaml | 1 - roles/ensure-wheel/tasks/main.yaml | 15 --- test-playbooks/ensure-pip.yaml | 8 ++ zuul-tests.d/python-jobs.yaml | 3 + zuul-tests.d/python-roles-jobs.yaml | 130 +------------------------- zuul.d/python-jobs.yaml | 2 +- 18 files changed, 48 insertions(+), 167 deletions(-) delete mode 100644 playbooks/python/release-pre.yaml delete mode 100644 roles/ensure-wheel/README.rst delete mode 100644 roles/ensure-wheel/defaults/main.yaml delete mode 100644 roles/ensure-wheel/tasks/main.yaml diff --git a/doc/source/python-roles.rst b/doc/source/python-roles.rst index 03d910f04..58cc4d11b 100644 --- a/doc/source/python-roles.rst +++ b/doc/source/python-roles.rst @@ -11,7 +11,6 @@ Python Roles .. zuul:autorole:: ensure-tox .. zuul:autorole:: ensure-twine .. zuul:autorole:: ensure-virtualenv -.. zuul:autorole:: ensure-wheel .. zuul:autorole:: fetch-coverage-output .. zuul:autorole:: fetch-python-sdist-output .. zuul:autorole:: fetch-sphinx-output diff --git a/playbooks/python/release-pre.yaml b/playbooks/python/release-pre.yaml deleted file mode 100644 index 409944ed0..000000000 --- a/playbooks/python/release-pre.yaml +++ /dev/null @@ -1,4 +0,0 @@ -- hosts: all - roles: - - role: ensure-wheel - wheel_python: "{{ release_python | default(python) }}" diff --git a/roles/build-python-release/README.rst b/roles/build-python-release/README.rst index 7e16f3a16..e67681d37 100644 --- a/roles/build-python-release/README.rst +++ b/roles/build-python-release/README.rst @@ -5,7 +5,8 @@ Build sdist and wheel for Python projects. .. zuul:rolevar:: release_python :default: python - The python interpreter to use. Defaults to "python". + The python interpreter to use. Set it to "python3" to use python 3, + for example. .. zuul:rolevar:: bdist_wheel_xargs :default: '' diff --git a/roles/ensure-pip/README.rst b/roles/ensure-pip/README.rst index 9a9951340..bcc5fcb3e 100644 --- a/roles/ensure-pip/README.rst +++ b/roles/ensure-pip/README.rst @@ -11,6 +11,9 @@ platforms. On some platforms it may install the package under the Python 2 interpreter and in others Python 3. You should use a qualified name (``pip2`` or ``pip3``) to avoid confusion. +This role will also ``wheel`` components sufficient to run +``bdist_wheel`` builds or ``pip wheel`` on a source tree. + **Role Variables** .. zuul:rolevar:: ensure_pip_from_packages diff --git a/roles/ensure-pip/tasks/Debian.yaml b/roles/ensure-pip/tasks/Debian.yaml index 95b292a6f..e9640752b 100644 --- a/roles/ensure-pip/tasks/Debian.yaml +++ b/roles/ensure-pip/tasks/Debian.yaml @@ -4,6 +4,7 @@ - python3-pip - python3-setuptools - python3-venv + - python3-wheel become: yes - name: Install Python 2 pip @@ -11,5 +12,6 @@ name: - python-setuptools - python-pip + - python-wheel become: yes when: ensure_pip_from_packages_with_python2 diff --git a/roles/ensure-pip/tasks/Gentoo.yaml b/roles/ensure-pip/tasks/Gentoo.yaml index 67d129940..562db8d4d 100644 --- a/roles/ensure-pip/tasks/Gentoo.yaml +++ b/roles/ensure-pip/tasks/Gentoo.yaml @@ -1,11 +1,15 @@ - name: Install Python 3 pip package: - name: dev-python/pip + name: + - dev-python/pip + - dev-python/wheel become: yes - name: Install Python 2 pip package: - name: dev-python/pip + name: + - dev-python/pip + - dev-python/wheel become: yes when: - ensure_pip_from_packages_with_python2 diff --git a/roles/ensure-pip/tasks/RedHat.yaml b/roles/ensure-pip/tasks/RedHat.yaml index 3259fbf5a..a9a6a405d 100644 --- a/roles/ensure-pip/tasks/RedHat.yaml +++ b/roles/ensure-pip/tasks/RedHat.yaml @@ -3,9 +3,8 @@ name: - python3-pip - python3-setuptools + - python3-wheel state: present - when: - - ansible_python.version.major == 3 become: yes - name: Install Python 2 pip @@ -14,7 +13,9 @@ - python-pip - python-setuptools - python-virtualenv + - python-wheel state: present enablerepo: epel become: yes - when: ensure_pip_from_packages_with_python2 + when: ensure_pip_from_packages_with_python2 or + ansible_python.version.major == 2 diff --git a/roles/ensure-pip/tasks/Suse.yaml b/roles/ensure-pip/tasks/Suse.yaml index 8eb1109d8..efe09e58d 100644 --- a/roles/ensure-pip/tasks/Suse.yaml +++ b/roles/ensure-pip/tasks/Suse.yaml @@ -1,10 +1,14 @@ - name: Install Python 3 pip package: - name: python3-pip + name: + - python3-pip + - python3-wheel become: yes - name: Install Python 2 pip package: - name: python2-pip + name: + - python2-pip + - python2-wheel become: yes when: ensure_pip_from_packages_with_python2 diff --git a/roles/ensure-pip/tasks/main.yaml b/roles/ensure-pip/tasks/main.yaml index 06374e01d..e85d2ab5d 100644 --- a/roles/ensure-pip/tasks/main.yaml +++ b/roles/ensure-pip/tasks/main.yaml @@ -17,9 +17,11 @@ # is present. if [ "$PYTHON2" -eq "1" ] ; then command -v pip2 || command -v pip || exit 1 + python2 -m wheel --help || exit 1 fi if [ "$PYTHON3" -eq "1" ] ; then command -v pip3 || command -v pip || exit 1 + python3 -m wheel --help || exit 1 fi args: executable: /bin/bash diff --git a/roles/ensure-pip/tasks/source.yaml b/roles/ensure-pip/tasks/source.yaml index 9c0bfa794..2f9593304 100644 --- a/roles/ensure-pip/tasks/source.yaml +++ b/roles/ensure-pip/tasks/source.yaml @@ -16,6 +16,13 @@ loop_control: loop_var: zj_item +- name: 'Install wheel' + command: '{{ zj_item }} -m pip install wheel' + become: yes + loop: '{{ ensure_pip_from_upstream_interpreters }}' + loop_control: + loop_var: zj_item + - name: Remove temporary install dir file: state: absent diff --git a/roles/ensure-pip/tasks/xenial.yaml b/roles/ensure-pip/tasks/xenial.yaml index 6e7817ebf..987974a1c 100644 --- a/roles/ensure-pip/tasks/xenial.yaml +++ b/roles/ensure-pip/tasks/xenial.yaml @@ -12,6 +12,7 @@ - python3-pip - python3-setuptools - python3-venv + - python3-wheel become: yes - name: Install Python 2 pip @@ -19,6 +20,7 @@ name: - python-setuptools - python-pip + - python-wheel become: yes when: ensure_pip_from_packages_with_python2 diff --git a/roles/ensure-wheel/README.rst b/roles/ensure-wheel/README.rst deleted file mode 100644 index f3f49dc07..000000000 --- a/roles/ensure-wheel/README.rst +++ /dev/null @@ -1,9 +0,0 @@ -Ensure wheel exists for Python projects. - -**Role Variables** - -.. zuul:rolevar:: wheel_python - :default: python3 - - The python interpreter to use. Defaults to "python3". - diff --git a/roles/ensure-wheel/defaults/main.yaml b/roles/ensure-wheel/defaults/main.yaml deleted file mode 100644 index 8c6056d3b..000000000 --- a/roles/ensure-wheel/defaults/main.yaml +++ /dev/null @@ -1 +0,0 @@ -wheel_python: python3 diff --git a/roles/ensure-wheel/tasks/main.yaml b/roles/ensure-wheel/tasks/main.yaml deleted file mode 100644 index cf3fa8690..000000000 --- a/roles/ensure-wheel/tasks/main.yaml +++ /dev/null @@ -1,15 +0,0 @@ -- name: Check for wheel - command: "{{ wheel_python }} -m wheel" - failed_when: false - register: wheel_exists - -- name: Include ensure-pip - vars: - ensure_pip_from_upstream_interpreters: "{{ wheel_python }}" - include_role: - name: ensure-pip - when: wheel_exists.rc != 0 - -- name: Install wheel - command: "{{ wheel_python }} -m pip install --user wheel" - when: wheel_exists.rc != 0 diff --git a/test-playbooks/ensure-pip.yaml b/test-playbooks/ensure-pip.yaml index 0cba62a66..f8eaa529a 100644 --- a/test-playbooks/ensure-pip.yaml +++ b/test-playbooks/ensure-pip.yaml @@ -26,6 +26,14 @@ path: '{{ _tmp_venv.path }}' state: absent + - name: Sanity check pip wheel generation + shell: | + cd {{ ansible_user_dir }}/src/opendev.org/zuul/zuul + # This should run anywhere without too much logic ... + run_pip=$(command -v pip3 || command -v pip2 || command -v pip) + $run_pip wheel --no-deps . + ls zuul-*.whl || exit 1 + # ensure-virtualenv - name: Include ensure-virtualenv include_role: diff --git a/zuul-tests.d/python-jobs.yaml b/zuul-tests.d/python-jobs.yaml index fcf69afdf..fdb2c2726 100644 --- a/zuul-tests.d/python-jobs.yaml +++ b/zuul-tests.d/python-jobs.yaml @@ -1,6 +1,9 @@ - job: name: zuul-jobs-test-ensure-pip description: Test the ensure-pip role + # dummy project for wheel build test + required-projects: + - zuul/zuul files: - roles/ensure-pip/.* - roles/ensure-virtualenv/.* diff --git a/zuul-tests.d/python-roles-jobs.yaml b/zuul-tests.d/python-roles-jobs.yaml index 4da6fec6c..40ee3256c 100644 --- a/zuul-tests.d/python-roles-jobs.yaml +++ b/zuul-tests.d/python-roles-jobs.yaml @@ -8,7 +8,7 @@ vars: role_name: build-python-release release_python: python3 - pre-run: playbooks/python/release-pre.yaml + pre-run: playbooks/python/pre.yaml run: test-playbooks/simple-role-test.yaml tags: all-platforms @@ -103,123 +103,14 @@ - name: ubuntu-xenial label: ubuntu-xenial -- job: - name: zuul-jobs-test-ensure-wheel-jobs - description: Test the ensure-wheel role - files: - - roles/ensure-wheel/.* - - setup.cfg - - setup.py - vars: - role_name: ensure-wheel - run: test-playbooks/simple-role-test.yaml - tags: all-platforms - -- job: - name: zuul-jobs-test-ensure-wheel-jobs-centos-7 - description: Test the ensure-wheel role on centos-7 - parent: zuul-jobs-test-ensure-wheel-jobs - tags: auto-generated - nodeset: - nodes: - - name: centos-7 - label: centos-7 - -- job: - name: zuul-jobs-test-ensure-wheel-jobs-centos-8 - description: Test the ensure-wheel role on centos-8 - parent: zuul-jobs-test-ensure-wheel-jobs - tags: auto-generated - nodeset: - nodes: - - name: centos-8 - label: centos-8 - -- job: - name: zuul-jobs-test-ensure-wheel-jobs-debian-stretch - description: Test the ensure-wheel role on debian-stretch - parent: zuul-jobs-test-ensure-wheel-jobs - tags: auto-generated - nodeset: - nodes: - - name: debian-stretch - label: debian-stretch - -- job: - name: zuul-jobs-test-ensure-wheel-jobs-fedora-31 - description: Test the ensure-wheel role on fedora-31 - parent: zuul-jobs-test-ensure-wheel-jobs - tags: auto-generated - nodeset: - nodes: - - name: fedora-31 - label: fedora-31 - -- job: - name: zuul-jobs-test-ensure-wheel-jobs-gentoo-17-0-systemd - description: Test the ensure-wheel role on gentoo-17-0-systemd - parent: zuul-jobs-test-ensure-wheel-jobs - tags: auto-generated - nodeset: - nodes: - - name: gentoo-17-0-systemd - label: gentoo-17-0-systemd - -- job: - name: zuul-jobs-test-ensure-wheel-jobs-opensuse-15 - description: Test the ensure-wheel role on opensuse-15 - parent: zuul-jobs-test-ensure-wheel-jobs - tags: auto-generated - nodeset: - nodes: - - name: opensuse-15 - label: opensuse-15 - -- job: - name: zuul-jobs-test-ensure-wheel-jobs-opensuse-tumbleweed-nv - voting: false - description: Test the ensure-wheel role on opensuse-tumbleweed - parent: zuul-jobs-test-ensure-wheel-jobs - tags: auto-generated - nodeset: - nodes: - - name: opensuse-tumbleweed - label: opensuse-tumbleweed - -- job: - name: zuul-jobs-test-ensure-wheel-jobs-ubuntu-bionic - description: Test the ensure-wheel role on ubuntu-bionic - parent: zuul-jobs-test-ensure-wheel-jobs - tags: auto-generated - nodeset: - nodes: - - name: ubuntu-bionic - label: ubuntu-bionic - -- job: - name: zuul-jobs-test-ensure-wheel-jobs-ubuntu-xenial - description: Test the ensure-wheel role on ubuntu-xenial - parent: zuul-jobs-test-ensure-wheel-jobs - tags: auto-generated - nodeset: - nodes: - - name: ubuntu-xenial - label: ubuntu-xenial - - job: name: zuul-jobs-test-ensure-python-pyenv description: Test the ensure-python role with pyenv files: - - roles/ensure-python/.* - - test-playbooks/ensure-python-pyenv.yaml + - zuul-tests.d/python-roles-jobs.yaml run: test-playbooks/ensure-python-pyenv.yaml tags: all-platforms -# -* AUTOGENERATED *- -# The following project section is autogenerated by -# tools/update-test-platforms.py -# Please re-run to generate new job lists - - job: name: zuul-jobs-test-ensure-python-pyenv-centos-7 description: Test the ensure-python role with pyenv on centos-7 @@ -323,15 +214,6 @@ - zuul-jobs-test-build-python-release-jobs-opensuse-tumbleweed-nv - zuul-jobs-test-build-python-release-jobs-ubuntu-bionic - zuul-jobs-test-build-python-release-jobs-ubuntu-xenial - - zuul-jobs-test-ensure-wheel-jobs-centos-7 - - zuul-jobs-test-ensure-wheel-jobs-centos-8 - - zuul-jobs-test-ensure-wheel-jobs-debian-stretch - - zuul-jobs-test-ensure-wheel-jobs-fedora-31 - - zuul-jobs-test-ensure-wheel-jobs-gentoo-17-0-systemd - - zuul-jobs-test-ensure-wheel-jobs-opensuse-15 - - zuul-jobs-test-ensure-wheel-jobs-opensuse-tumbleweed-nv - - zuul-jobs-test-ensure-wheel-jobs-ubuntu-bionic - - zuul-jobs-test-ensure-wheel-jobs-ubuntu-xenial - zuul-jobs-test-ensure-python-pyenv-centos-7 - zuul-jobs-test-ensure-python-pyenv-centos-8 - zuul-jobs-test-ensure-python-pyenv-debian-stretch @@ -351,14 +233,6 @@ - zuul-jobs-test-build-python-release-jobs-opensuse-15 - zuul-jobs-test-build-python-release-jobs-ubuntu-bionic - zuul-jobs-test-build-python-release-jobs-ubuntu-xenial - - zuul-jobs-test-ensure-wheel-jobs-centos-7 - - zuul-jobs-test-ensure-wheel-jobs-centos-8 - - zuul-jobs-test-ensure-wheel-jobs-debian-stretch - - zuul-jobs-test-ensure-wheel-jobs-fedora-31 - - zuul-jobs-test-ensure-wheel-jobs-gentoo-17-0-systemd - - zuul-jobs-test-ensure-wheel-jobs-opensuse-15 - - zuul-jobs-test-ensure-wheel-jobs-ubuntu-bionic - - zuul-jobs-test-ensure-wheel-jobs-ubuntu-xenial - zuul-jobs-test-ensure-python-pyenv-centos-7 - zuul-jobs-test-ensure-python-pyenv-centos-8 - zuul-jobs-test-ensure-python-pyenv-debian-stretch diff --git a/zuul.d/python-jobs.yaml b/zuul.d/python-jobs.yaml index 204069969..db4cc61d4 100644 --- a/zuul.d/python-jobs.yaml +++ b/zuul.d/python-jobs.yaml @@ -249,7 +249,7 @@ name: build-python-release description: | Build a source tarball and a bdist wheel for uploading. - pre-run: playbooks/python/release-pre.yaml + pre-run: playbooks/python/pre.yaml run: playbooks/python/release.yaml post-run: playbooks/python/tarball-post.yaml