Bump up to python3.10

This adds python3.10 testing on Jammy and switches the docker images to
python3.10 from 3.8.

We run sudo for postgres with -Hi to avoid non fatal errors when
postres' client attempts to write command history to Zuul's homedir (it
is running as the postgres user which can't write to zuul's homedir). We
also need to update the libffi package version for jammy to 8 in
bindep.txt. Finally, python_version values need to be quoted as "3.10"
is different than 3.10 which is equivalent to 3.1 when serialized by
yaml as a float.

Force setuptools to use stdlib (shipped by the distro) distutils to
avoid problems with virtualenvs not actually being virtualenvs.

Finally we switch the bulk of jobs over to using nodeset: ubuntu-jammy
as the default python there is 3.10.

Change-Id: I97b90bb7a23c90f108f23dda9fdd0e89f9f4dbca

.zuul.yaml

@@ -6,11 +6,11 @@
       # need their python versions updated when these labels change to
       # a platform that uses a different python version.
       - name: controller
-        label: ubuntu-focal
+        label: ubuntu-jammy
       - name: node1
-        label: ubuntu-focal
+        label: ubuntu-jammy
       - name: node2
-        label: ubuntu-focal
+        label: ubuntu-jammy
     groups:
       - name: node
         nodes:
@@ -56,7 +56,7 @@
     description: |
       Zuul unit tests with ZooKeeper running
     parent: tox
-    nodeset: ubuntu-focal
+    nodeset: ubuntu-jammy
     pre-run: playbooks/zuul-tox/pre.yaml
     post-run: playbooks/zuul-tox/post-system-logs.yaml
     vars:
@@ -73,6 +73,7 @@
 - job:
     name: zuul-tox-remote
     parent: tox
+    nodeset: ubuntu-jammy
     timeout: 2700  # 45 minutes
     pre-run: playbooks/zuul-tox/pre.yaml
     post-run: playbooks/zuul-tox/post-system-logs.yaml
@@ -99,12 +100,12 @@
       tox_envlist: zuul_client
 
 - job:
-    name: zuul-tox-py39
+    name: zuul-tox-py310
     parent: zuul-tox
     timeout: 5400  # 90 minutes
     vars:
-      tox_envlist: py39
-      python_version: 3.9
+      tox_envlist: py310
+      python_version: "3.10"
 
 - job:
     name: zuul-tox-py38
@@ -112,11 +113,12 @@
     timeout: 5400  # 90 minutes
     vars:
       tox_envlist: py38
-      python_version: 3.8
+      python_version: "3.8"
+    nodeset: ubuntu-focal
 
 - job:
-    name: zuul-tox-py39-multi-scheduler
-    parent: zuul-tox-py39
+    name: zuul-tox-py310-multi-scheduler
+    parent: zuul-tox-py310
     voting: false
     vars:
       tox_environment:
@@ -135,6 +137,7 @@
       zuul_work_dir: "{{ zuul.project.src_dir }}/web"
       create_tarball_directory: build
     run: playbooks/dashboard/run.yaml
+    nodeset: ubuntu-jammy
 
 - job:
     name: zuul-build-dashboard-openstack-whitelabel
@@ -183,6 +186,7 @@
     post-run: playbooks/tutorial/post.yaml
     required-projects:
       - zuul/zuul
+    nodeset: ubuntu-jammy
 
 # Image building jobs
 - secret:
@@ -208,8 +212,8 @@
     allowed-projects: zuul/zuul
     timeout: 2700  # 45 minutes
     requires:
-      - python-builder-3.8-bullseye-container-image
-      - python-base-3.8-bullseye-container-image
+      - python-builder-3.10-bullseye-container-image
+      - python-base-3.10-bullseye-container-image
     provides: zuul-container-image
     vars: &zuul_image_vars
       docker_images:
@@ -247,8 +251,8 @@
     description: Build Docker images and upload to Docker Hub.
     allowed-projects: zuul/zuul
     requires:
-      - python-builder-3.8-bullseye-container-image
-      - python-base-3.8-bullseye-container-image
+      - python-builder-3.10-bullseye-container-image
+      - python-base-3.10-bullseye-container-image
     provides: zuul-container-image
     secrets:
       name: docker_credentials
@@ -273,11 +277,13 @@
     name: zuul-build-python-release
     parent: build-python-release
     pre-run: playbooks/release/pre.yaml
+    nodeset: ubuntu-jammy
 
 - job:
     name: zuul-release-python
     parent: opendev-release-python
     pre-run: playbooks/release/pre.yaml
+    nodeset: ubuntu-jammy
 
 - project:
     vars:
@@ -290,20 +296,23 @@
         - tox-linters:
             vars:
               tox_install_bindep: false
+            nodeset: ubuntu-jammy
         - zuul-tox-py38
-        - zuul-tox-py39
-        - zuul-tox-py39-multi-scheduler
+        - zuul-tox-py310
+        - zuul-tox-py310-multi-scheduler
         - zuul-build-dashboard-openstack-whitelabel
         - zuul-build-dashboard-software-factory
         - zuul-build-dashboard-opendev
         - nodejs-run-lint:
             vars:
               zuul_work_dir: "{{ zuul.project.src_dir }}/web"
+            nodeset: ubuntu-jammy
         - nodejs-run-test:
             vars:
               zuul_work_dir: "{{ zuul.project.src_dir }}/web"
             files:
               - web/.*
+            nodeset: ubuntu-jammy
         - zuul-stream-functional-2.8
         - zuul-stream-functional-2.9
         - zuul-stream-functional-5
@@ -320,17 +329,20 @@
         - tox-linters:
             vars:
               tox_install_bindep: false
+            nodeset: ubuntu-jammy
         - zuul-tox-py38
-        - zuul-tox-py39
+        - zuul-tox-py310
         - zuul-build-dashboard
         - nodejs-run-lint:
             vars:
               zuul_work_dir: "{{ zuul.project.src_dir }}/web"
+            nodeset: ubuntu-jammy
         - nodejs-run-test:
             vars:
               zuul_work_dir: "{{ zuul.project.src_dir }}/web"
             files:
               - web/.*
+            nodeset: ubuntu-jammy
         - zuul-stream-functional-2.8
         - zuul-stream-functional-2.9
         - zuul-stream-functional-5

Dockerfile

@@ -19,7 +19,7 @@ COPY web /tmp/src
 # Explicitly run the Javascript build
 RUN cd /tmp/src && yarn install -d && yarn build
 
-FROM docker.io/opendevorg/python-builder:3.8-bullseye as builder
+FROM docker.io/opendevorg/python-builder:3.10-bullseye as builder
 ENV DEBIAN_FRONTEND=noninteractive
 
 # Optional location of Zuul API endpoint.
@@ -48,7 +48,7 @@ RUN /output/install-from-bindep \
   && echo $OPENSHIFT_SHA /tmp/openshift-install/openshift-client.tgz | sha256sum --check \
   && tar xvfz openshift-client.tgz --strip-components=1 -C /tmp/openshift-install
 
-FROM docker.io/opendevorg/python-base:3.8-bullseye as zuul
+FROM docker.io/opendevorg/python-base:3.10-bullseye as zuul
 ENV DEBIAN_FRONTEND=noninteractive
 
 COPY --from=builder /output/ /output

bindep.txt

@@ -25,8 +25,9 @@ libssl1.1 [platform:debian platform:ubuntu-bionic]
 libssl1.0.0 [platform:ubuntu-xenial]
 libffi-dev [compile test platform:dpkg platform:apk]
 libffi [platform:apk]
+libffi8 [platform:ubuntu-jammy]
 libffi7 [platform:ubuntu-focal platform:debian-bullseye]
-libffi6 [platform:dpkg !platform:ubuntu-focal !platform:debian-bullseye]
+libffi6 [platform:dpkg !platform:ubuntu-focal !platform:ubuntu-jammy !platform:debian-bullseye]
 libffi-devel [compile test platform:rpm]
 libyaml-0-2 [platform:dpkg platform:suse]
 libyaml [platform:redhat]

playbooks/zuul-stream/functional.yaml

@@ -5,7 +5,7 @@
         # This value is used by Ansible to find the zuul.ansible code
         # that Zuul's ansible plugins consume. It must be updated when
         # the python version of the platform is changed.
-        python_path: "/usr/local/lib/python3.8/dist-packages"
+        python_path: "/usr/local/lib/python3.10/dist-packages"
 
     - name: Run ansible that should succeed
       command: >

playbooks/zuul-stream/pre.yaml

@@ -28,6 +28,13 @@
     - name: Install managed ansible versions
       command: /usr/local/bin/zuul-manage-ansible -v
       become: yes
+      environment:
+        # This appears to be necessary because without it the virtualenvs
+        # that virtualenv creates are not actually virtualenvs. They are just
+        # aliases for the root python installation instead. Then we try to
+        # install multiple ansible versions to the root (via multiple fake
+        # venvs) and the installation fails due to conflicts.
+        SETUPTOOLS_USE_DISTUTILS: stdlib
 
     - name: Copy inventory
       copy:

setup.cfg

@@ -16,6 +16,7 @@ classifier =
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
 
 [pbr]
 warnerrors = True

tools/test-setup.sh

@@ -44,8 +44,8 @@ mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e "
     CREATE DATABASE openstack_citest CHARACTER SET utf8;"
 
 # setup postgres user and database
-sudo -u postgres psql -c "CREATE ROLE $DB_USER WITH LOGIN SUPERUSER PASSWORD '$DB_PW';"
-sudo -u postgres psql -c "CREATE DATABASE openstack_citest OWNER $DB_USER TEMPLATE template0 ENCODING 'UTF8';"
+sudo -Hi -u postgres psql -c "CREATE ROLE $DB_USER WITH LOGIN SUPERUSER PASSWORD '$DB_PW';"
+sudo -Hi -u postgres psql -c "CREATE DATABASE openstack_citest OWNER $DB_USER TEMPLATE template0 ENCODING 'UTF8';"
 
 LSBDISTCODENAME=$(lsb_release -cs)
 if [ $LSBDISTCODENAME == 'xenial' ]; then