nodepool-elements: Use venv for utilities

Since all platforms have Python 3, use the new ensure-venv element
from the dependent change to install bindep and os-testr.

Since we are no longer using pip to install anything during the
builder, this drops the dependency on pip-and-virtualenv from
nodepool-base.  Avoiding this element is our long-term goal, as it's
modification to system state are problematic in a number of ways.  To
maintain the status-quo, the pip-and-virtualenv element is added
explicitly to each build's element list, with a note on it's future.

The current plan for backwards compatability is to replicate the
environment pip-and-virtualenv provides in a base role/job that can be
optionally included.  To test this, provide a new node type
"ubuntu-bionic-plain" that will not include the pip-and-virtualenv
element.  This is put on just one provider (rax) to minimise impact.

The dependent-change (and a dib release) is required before merge so
the ensure-venv element is available.

Depends-On: https://review.opendev.org/707513
Change-Id: I85438baf5bb31790a56fe5b38327361f0a2398e9

nodepool/elements/infra-package-needs/element-deps

@@ -1,2 +1,3 @@
+ensure-venv
 openssh-server
 package-installs

nodepool/elements/infra-package-needs/install.d/40-install-bindep

@@ -22,5 +22,5 @@ if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
 fi
 set -e
 
-$DIB_PYTHON_VIRTUALENV /usr/bindep-env
+python3 -m venv /usr/bindep-env
 /usr/bindep-env/bin/pip install bindep

nodepool/elements/nodepool-base/element-deps

@@ -1,3 +1,3 @@
+ensure-venv
 package-installs
-pip-and-virtualenv
 zuul-worker

nodepool/elements/nodepool-base/install.d/91-venv-os-testr

@@ -18,5 +18,5 @@ if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
 fi
 set -e
 
-$DIB_PYTHON_VIRTUALENV /usr/os-testr-env
+python3 -m venv /usr/os-testr-env
 /usr/os-testr-env/bin/pip install os-testr

nodepool/nl01.openstack.org.yaml

@@ -34,6 +34,7 @@ labels:
     min-ready: 10
   - name: ubuntu-xenial
     min-ready: 1
+  - name: ubuntu-bionic-plain
 
 providers:
   - name: rax-dfw
@@ -63,6 +64,8 @@ providers:
         config-drive: true
       - name: ubuntu-xenial
         config-drive: true
+      - name: ubuntu-bionic-plain
+        config-drive: true
     pools:
       - name: main
         max-servers: 140
@@ -112,6 +115,11 @@ providers:
             flavor-name: 'Performance'
             diskimage: ubuntu-bionic
             key-name: infra-root-keys-2018-06-15
+          - name: ubuntu-bionic-plain
+            min-ram: 8192
+            flavor-name: 'Performance'
+            diskimage: ubuntu-bionic-plain
+            key-name: infra-root-keys-2018-06-15
           - name: ubuntu-xenial
             min-ram: 8192
             flavor-name: 'Performance'
@@ -154,3 +162,4 @@ diskimages:
   - name: opensuse-tumbleweed
   - name: ubuntu-bionic
   - name: ubuntu-xenial
+  - name: ubuntu-bionic-plain

nodepool/nodepool.yaml

@@ -72,19 +72,43 @@ providers:
     region-name: 'DFW'
     cloud: rax
     rate: 0.001
-    diskimages: *provider_diskimages
+    diskimages: &rax_provider_diskimages
+      # NOTE(ianw) 2020-02-17 : duplicated to add ubuntu-bionic-plain
+      # while we work to remove pip-and-virtualenv
+      - name: centos-7
+        config-drive: true
+      - name: centos-8
+        config-drive: true
+      - name: debian-buster
+        config-drive: true
+      - name: debian-stretch
+        config-drive: true
+      - name: fedora-30
+        config-drive: true
+      - name: gentoo-17-0-systemd
+        config-drive: true
+      - name: opensuse-15
+        config-drive: true
+      - name: opensuse-tumbleweed
+        config-drive: true
+      - name: ubuntu-bionic
+        config-drive: true
+      - name: ubuntu-bionic-plain
+        config-drive: true
+      - name: ubuntu-xenial
+        config-drive: true
 
   - name: rax-iad
     region-name: 'IAD'
     cloud: rax
     rate: 0.001
-    diskimages: *provider_diskimages
+    diskimages: *rax_provider_diskimages
 
   - name: rax-ord
     region-name: 'ORD'
     cloud: rax
     rate: 0.001
-    diskimages: *provider_diskimages
+    diskimages: *rax_provider_diskimages
 
   - name: vexxhost-ca-ymq-1
     region-name: 'ca-ymq-1'
@@ -129,6 +153,9 @@ diskimages:
       - growroot
       - infra-package-needs
       - epel
+      # NOTE(ianw) 2020-02-17 : we are working to move this into job
+      # configuration
+      - pip-and-virtualenv
     env-vars:
       TMPDIR: /opt/dib_tmp
       DIB_CHECKSUM: '1'
@@ -152,6 +179,9 @@ diskimages:
       - growroot
       - infra-package-needs
       - epel
+      # NOTE(ianw) 2020-02-17 : we are working to move this into job
+      # configuration
+      - pip-and-virtualenv
     env-vars:
       TMPDIR: /opt/dib_tmp
       DIB_CHECKSUM: '1'
@@ -174,6 +204,9 @@ diskimages:
       - cache-devstack
       - growroot
       - infra-package-needs
+      # NOTE(ianw) 2020-02-17 : we are working to move this into job
+      # configuration
+      - pip-and-virtualenv
     release: stretch
     env-vars:
       TMPDIR: /opt/dib_tmp
@@ -202,6 +235,9 @@ diskimages:
       - cache-devstack
       - growroot
       - infra-package-needs
+      # NOTE(ianw) 2020-02-17 : we are working to move this into job
+      # configuration
+      - pip-and-virtualenv
     release: buster
     env-vars:
       TMPDIR: /opt/dib_tmp
@@ -237,6 +273,7 @@ diskimages:
                     fstab:
                       options: "defaults,nobarrier,noatime"
                       fsck-passno: 1
+
   - name: fedora-30
     pause: true
     python-path: /usr/bin/python3
@@ -250,6 +287,9 @@ diskimages:
       - initialize-urandom
       - growroot
       - infra-package-needs
+      # NOTE(ianw) 2020-02-17 : we are working to move this into job
+      # configuration
+      - pip-and-virtualenv
     release: 30
     env-vars:
       TMPDIR: /opt/dib_tmp
@@ -270,6 +310,9 @@ diskimages:
       - nodepool-base
       - growroot
       - infra-package-needs
+      # NOTE(ianw) 2020-02-17 : we are working to move this into job
+      # configuration
+      - pip-and-virtualenv
     env-vars:
       TMPDIR: /opt/dib_tmp
       DIB_CHECKSUM: '1'
@@ -293,6 +336,9 @@ diskimages:
       - cache-devstack
       - growroot
       - infra-package-needs
+      # NOTE(ianw) 2020-02-17 : we are working to move this into job
+      # configuration
+      - pip-and-virtualenv
     release: '15.1'
     env-vars:
       TMPDIR: /opt/dib_tmp
@@ -315,6 +361,9 @@ diskimages:
       - cache-devstack
       - growroot
       - infra-package-needs
+      # NOTE(ianw) 2020-02-17 : we are working to move this into job
+      # configuration
+      - pip-and-virtualenv
     release: 'tumbleweed'
     env-vars:
       TMPDIR: /opt/dib_tmp
@@ -337,6 +386,9 @@ diskimages:
       - cache-devstack
       - growroot
       - infra-package-needs
+      # NOTE(ianw) 2020-02-17 : we are working to move this into job
+      # configuration
+      - pip-and-virtualenv
     release: bionic
     env-vars:
       TMPDIR: /opt/dib_tmp
@@ -364,6 +416,9 @@ diskimages:
       - initialize-urandom
       - growroot
       - infra-package-needs
+      # NOTE(ianw) 2020-02-17 : we are working to move this into job
+      # configuration
+      - pip-and-virtualenv
     release: xenial
     env-vars:
       TMPDIR: /opt/dib_tmp
@@ -379,3 +434,32 @@ diskimages:
       GIT_HTTP_LOW_SPEED_TIME: '300'
       GIT_HTTP_LOW_SPEED_LIMIT: '1000'
       DIB_SHOW_IMAGE_USAGE: '1'
+  # NOTE(ianw) 2020-02-17 : this is an alternative image without the
+  # pip-and-virtualenv element we can test ansible roles to achieve
+  # full backwards-compatability with.
+  - name: ubuntu-bionic-plain
+    pause: false
+    elements:
+      - ubuntu-minimal
+      - vm
+      - simple-init
+      - openstack-repos
+      - nodepool-base
+      - cache-devstack
+      - growroot
+      - infra-package-needs
+    release: bionic
+    env-vars:
+      TMPDIR: /opt/dib_tmp
+      DIB_CHECKSUM: '1'
+      DIB_IMAGE_CACHE: /opt/dib_cache
+      DIB_JOURNAL_SIZE: '512'
+      DIB_APT_LOCAL_CACHE: '0'
+      DIB_DISABLE_APT_CLEANUP: '1'
+      DIB_GRUB_TIMEOUT: '0'
+      DIB_DISTRIBUTION_MIRROR: 'http://mirror.dfw.rax.openstack.org/ubuntu'
+      DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg'
+      DIB_DEBIAN_COMPONENTS: 'main,universe'
+      GIT_HTTP_LOW_SPEED_TIME: '300'
+      GIT_HTTP_LOW_SPEED_LIMIT: '1000'
+      DIB_SHOW_IMAGE_USAGE: '1'