More narrowly tailor the ensure-pip Debian workaround

The docker image that we build the zuul executor from is a Debian
image, but it does not follow the same python3 policies as Debian
itself.  While we would not necessarily expect all roles to work
on the executor, it is reasonable to want to use the ensure-pip
role (which logically should be a no-op on the executor) for the
side effect of finding and returning the appropriate pip command.

Currently, the role fails on the executor because it mistakenly
concludes that it must install python3-venv to get a working
venv module.  By increasing the precision of the check for what
is missing (the actual error is a missing "ensurepip" python module
(oh irony!), we can avoid attempting an installation of
python3-venv on python docker images (including the Zuul executor
images).

This also adds the ensure-pip-localhost job

This tests that the ensure-pip role works on the Zuul executor.

The executor is a debian host with a working python environment,
so it should be a no-op (and no packages should need to be installed).

Change-Id: Id7f13f2f73d45e680f79c00a83751b185212a63d

roles/ensure-pip/tasks/workarounds.yaml

@@ -19,19 +19,19 @@
 
 # Part of this role is exporting a working virtualenv_command for you
 # -- on Debuntu, the presence of venv (i.e. "python3 -m venv --help"
-# works) doesn't actually mean venv works ... that's just a stub that
-# will give you an error telling you the python3-venv package isn't
-# installed.
+# works) doesn't actually mean venv works.  When "python3 -m venv foo"
+# is run, venv construction will fail because the "ensurepip" module
+# is not present.
 #
 # It's quite possible we have pip and so have skipped installing from
 # packages, where we would have brought this in.  To avoid requiring
 # sudo, which is the whole point of probing for pip and skipping
-# install if we have it, we probe for the package here and only
-# install if required.
-- name: Check for python3-venv
-  command: dpkg --status python3-venv
+# install if we have it, we probe for "ensurepip" here and only
+# install the package if required.
+- name: Check for ensurepip module
+  command: python3 -m ensurepip --help
   failed_when: false
-  register: _deb_venv_pkg
+  register: _ensurepip_module
   when:
     - ansible_os_family == 'Debian'
 
@@ -42,4 +42,4 @@
   become: yes
   when:
     - ansible_os_family == 'Debian'
-    - _deb_venv_pkg.rc != 0
+    - _ensurepip_module.rc != 0

test-playbooks/ensure-pip-localhost.yaml

@@ -0,0 +1,72 @@
+- hosts: localhost
+  tasks:
+    # ensure-pip
+
+    - name: Include ensure-pip
+      include_role:
+        name: ensure-pip
+
+    - name: Create temp directory
+      tempfile:
+        state: directory
+        suffix: venv-test
+      register: _tmp_venv
+
+    - name: Sanity check provided virtualenv command installs
+      pip:
+        name: tox
+        virtualenv_command: '{{ ensure_pip_virtualenv_command }}'
+        virtualenv: '{{ _tmp_venv.path }}'
+
+    - name: Sanity check installed command runs without error
+      command: '{{ _tmp_venv.path }}/bin/tox --version'
+
+    - name: Remove tmpdir
+      file:
+        path: '{{ _tmp_venv.path }}'
+        state: absent
+
+    - name: Sanity check pip wheel generation
+      shell: |
+        cd {{ ansible_user_dir }}/src/opendev.org/zuul/zuul-client
+        # This should run anywhere without too much logic ...
+        run_pip=$(command -v pip3 || command -v pip2 || command -v pip)
+        # Preinstall pbr to work around very old distutils lacking SNI support
+        $run_pip install pbr
+        $run_pip wheel --no-deps .
+        ls zuul_client-*.whl || exit 1
+
+    - name: Test virtualenv
+      # NOTE(ianw) 2022-02-03 : not supported on 9-stream, see inline comments
+      # NOTE(frickler) 2022-03-01 : pin pluggy so as to work on Debian Buster
+      when: not (ansible_facts['distribution'] == 'CentOS' and  ansible_facts['distribution_major_version']|int >= 9)
+      block:
+        # ensure-virtualenv
+        - name: Include ensure-virtualenv
+          include_role:
+            name: ensure-virtualenv
+
+        - name: Sanity check virtualenv command works
+          shell: |
+            tmp_venv=$(mktemp -d -t venv-XXXXXXXXXX)
+            trap "rm -rf $tmp_venv" EXIT
+            virtualenv $tmp_venv
+            $tmp_venv/bin/pip install tox "pluggy<1"
+          failed_when: false
+          register: _virtualenv_sanity
+
+        - name: Assert sanity check
+          fail:
+            msg: 'The virtualenv command does not appear to work!'
+          when:
+            - _virtualenv_sanity.rc != 0
+
+# NOTE(ianw) : this does not play nicely with pip-and-virtualenv which
+# has already installed from source.  We might be able to test this
+# once it's gone...
+
+# - hosts: all
+#   roles:
+#     - role: ensure-pip
+#       vars:
+#         ensure_pip_from_upstream: True

zuul-tests.d/python-jobs.yaml

@@ -110,6 +110,14 @@
         - name: ubuntu-jammy
           label: ubuntu-jammy
 
+- job:
+    name: zuul-jobs-test-ensure-pip-localhost
+    description: Test the ensure-pip role on the executor
+    parent: zuul-jobs-test-ensure-pip
+    run: test-playbooks/ensure-pip-localhost.yaml
+    nodeset:
+      nodes: []
+
 - job:
     name: zuul-jobs-test-ensure-sphinx
     description: Test the ensure-sphinx role
@@ -463,6 +471,7 @@
         - zuul-jobs-test-ensure-pip-ubuntu-bionic
         - zuul-jobs-test-ensure-pip-ubuntu-focal
         - zuul-jobs-test-ensure-pip-ubuntu-jammy
+        - zuul-jobs-test-ensure-pip-localhost
         - zuul-jobs-test-ensure-sphinx
         - zuul-jobs-test-ensure-tox-centos-7
         - zuul-jobs-test-ensure-tox-centos-8-stream