From d665053fe9b12bb237ffe810212772616a18ca64 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Tue, 3 May 2022 09:33:07 -0700 Subject: [PATCH] Bump up to python3.10 This adds python3.10 testing on Jammy and switches the docker images to python3.10 from 3.8. We run sudo for postgres with -Hi to avoid non fatal errors when postres' client attempts to write command history to Zuul's homedir (it is running as the postgres user which can't write to zuul's homedir). We also need to update the libffi package version for jammy to 8 in bindep.txt. Finally, python_version values need to be quoted as "3.10" is different than 3.10 which is equivalent to 3.1 when serialized by yaml as a float. Force setuptools to use stdlib (shipped by the distro) distutils to avoid problems with virtualenvs not actually being virtualenvs. Finally we switch the bulk of jobs over to using nodeset: ubuntu-jammy as the default python there is 3.10. Change-Id: I97b90bb7a23c90f108f23dda9fdd0e89f9f4dbca --- .zuul.yaml | 46 +++++++++++++++++---------- Dockerfile | 4 +-- bindep.txt | 3 +- playbooks/zuul-stream/functional.yaml | 2 +- playbooks/zuul-stream/pre.yaml | 7 ++++ setup.cfg | 1 + tools/test-setup.sh | 4 +-- 7 files changed, 44 insertions(+), 23 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 4caed89ec8..00d0a7053c 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -6,11 +6,11 @@ # need their python versions updated when these labels change to # a platform that uses a different python version. - name: controller - label: ubuntu-focal + label: ubuntu-jammy - name: node1 - label: ubuntu-focal + label: ubuntu-jammy - name: node2 - label: ubuntu-focal + label: ubuntu-jammy groups: - name: node nodes: @@ -56,7 +56,7 @@ description: | Zuul unit tests with ZooKeeper running parent: tox - nodeset: ubuntu-focal + nodeset: ubuntu-jammy pre-run: playbooks/zuul-tox/pre.yaml post-run: playbooks/zuul-tox/post-system-logs.yaml vars: @@ -73,6 +73,7 @@ - job: name: zuul-tox-remote parent: tox + nodeset: ubuntu-jammy timeout: 2700 # 45 minutes pre-run: playbooks/zuul-tox/pre.yaml post-run: playbooks/zuul-tox/post-system-logs.yaml @@ -99,12 +100,12 @@ tox_envlist: zuul_client - job: - name: zuul-tox-py39 + name: zuul-tox-py310 parent: zuul-tox timeout: 5400 # 90 minutes vars: - tox_envlist: py39 - python_version: 3.9 + tox_envlist: py310 + python_version: "3.10" - job: name: zuul-tox-py38 @@ -112,11 +113,12 @@ timeout: 5400 # 90 minutes vars: tox_envlist: py38 - python_version: 3.8 + python_version: "3.8" + nodeset: ubuntu-focal - job: - name: zuul-tox-py39-multi-scheduler - parent: zuul-tox-py39 + name: zuul-tox-py310-multi-scheduler + parent: zuul-tox-py310 voting: false vars: tox_environment: @@ -135,6 +137,7 @@ zuul_work_dir: "{{ zuul.project.src_dir }}/web" create_tarball_directory: build run: playbooks/dashboard/run.yaml + nodeset: ubuntu-jammy - job: name: zuul-build-dashboard-openstack-whitelabel @@ -183,6 +186,7 @@ post-run: playbooks/tutorial/post.yaml required-projects: - zuul/zuul + nodeset: ubuntu-jammy # Image building jobs - secret: @@ -208,8 +212,8 @@ allowed-projects: zuul/zuul timeout: 2700 # 45 minutes requires: - - python-builder-3.8-bullseye-container-image - - python-base-3.8-bullseye-container-image + - python-builder-3.10-bullseye-container-image + - python-base-3.10-bullseye-container-image provides: zuul-container-image vars: &zuul_image_vars docker_images: @@ -247,8 +251,8 @@ description: Build Docker images and upload to Docker Hub. allowed-projects: zuul/zuul requires: - - python-builder-3.8-bullseye-container-image - - python-base-3.8-bullseye-container-image + - python-builder-3.10-bullseye-container-image + - python-base-3.10-bullseye-container-image provides: zuul-container-image secrets: name: docker_credentials @@ -273,11 +277,13 @@ name: zuul-build-python-release parent: build-python-release pre-run: playbooks/release/pre.yaml + nodeset: ubuntu-jammy - job: name: zuul-release-python parent: opendev-release-python pre-run: playbooks/release/pre.yaml + nodeset: ubuntu-jammy - project: vars: @@ -290,20 +296,23 @@ - tox-linters: vars: tox_install_bindep: false + nodeset: ubuntu-jammy - zuul-tox-py38 - - zuul-tox-py39 - - zuul-tox-py39-multi-scheduler + - zuul-tox-py310 + - zuul-tox-py310-multi-scheduler - zuul-build-dashboard-openstack-whitelabel - zuul-build-dashboard-software-factory - zuul-build-dashboard-opendev - nodejs-run-lint: vars: zuul_work_dir: "{{ zuul.project.src_dir }}/web" + nodeset: ubuntu-jammy - nodejs-run-test: vars: zuul_work_dir: "{{ zuul.project.src_dir }}/web" files: - web/.* + nodeset: ubuntu-jammy - zuul-stream-functional-2.8 - zuul-stream-functional-2.9 - zuul-stream-functional-5 @@ -320,17 +329,20 @@ - tox-linters: vars: tox_install_bindep: false + nodeset: ubuntu-jammy - zuul-tox-py38 - - zuul-tox-py39 + - zuul-tox-py310 - zuul-build-dashboard - nodejs-run-lint: vars: zuul_work_dir: "{{ zuul.project.src_dir }}/web" + nodeset: ubuntu-jammy - nodejs-run-test: vars: zuul_work_dir: "{{ zuul.project.src_dir }}/web" files: - web/.* + nodeset: ubuntu-jammy - zuul-stream-functional-2.8 - zuul-stream-functional-2.9 - zuul-stream-functional-5 diff --git a/Dockerfile b/Dockerfile index d65a9746c4..c6cc17651c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ COPY web /tmp/src # Explicitly run the Javascript build RUN cd /tmp/src && yarn install -d && yarn build -FROM docker.io/opendevorg/python-builder:3.8-bullseye as builder +FROM docker.io/opendevorg/python-builder:3.10-bullseye as builder ENV DEBIAN_FRONTEND=noninteractive # Optional location of Zuul API endpoint. @@ -48,7 +48,7 @@ RUN /output/install-from-bindep \ && echo $OPENSHIFT_SHA /tmp/openshift-install/openshift-client.tgz | sha256sum --check \ && tar xvfz openshift-client.tgz --strip-components=1 -C /tmp/openshift-install -FROM docker.io/opendevorg/python-base:3.8-bullseye as zuul +FROM docker.io/opendevorg/python-base:3.10-bullseye as zuul ENV DEBIAN_FRONTEND=noninteractive COPY --from=builder /output/ /output diff --git a/bindep.txt b/bindep.txt index a16328e0fe..2e9c5e6962 100644 --- a/bindep.txt +++ b/bindep.txt @@ -25,8 +25,9 @@ libssl1.1 [platform:debian platform:ubuntu-bionic] libssl1.0.0 [platform:ubuntu-xenial] libffi-dev [compile test platform:dpkg platform:apk] libffi [platform:apk] +libffi8 [platform:ubuntu-jammy] libffi7 [platform:ubuntu-focal platform:debian-bullseye] -libffi6 [platform:dpkg !platform:ubuntu-focal !platform:debian-bullseye] +libffi6 [platform:dpkg !platform:ubuntu-focal !platform:ubuntu-jammy !platform:debian-bullseye] libffi-devel [compile test platform:rpm] libyaml-0-2 [platform:dpkg platform:suse] libyaml [platform:redhat] diff --git a/playbooks/zuul-stream/functional.yaml b/playbooks/zuul-stream/functional.yaml index fbc391ac0a..ee1643fbc2 100644 --- a/playbooks/zuul-stream/functional.yaml +++ b/playbooks/zuul-stream/functional.yaml @@ -5,7 +5,7 @@ # This value is used by Ansible to find the zuul.ansible code # that Zuul's ansible plugins consume. It must be updated when # the python version of the platform is changed. - python_path: "/usr/local/lib/python3.8/dist-packages" + python_path: "/usr/local/lib/python3.10/dist-packages" - name: Run ansible that should succeed command: > diff --git a/playbooks/zuul-stream/pre.yaml b/playbooks/zuul-stream/pre.yaml index cafdc01335..23fae3549f 100644 --- a/playbooks/zuul-stream/pre.yaml +++ b/playbooks/zuul-stream/pre.yaml @@ -28,6 +28,13 @@ - name: Install managed ansible versions command: /usr/local/bin/zuul-manage-ansible -v become: yes + environment: + # This appears to be necessary because without it the virtualenvs + # that virtualenv creates are not actually virtualenvs. They are just + # aliases for the root python installation instead. Then we try to + # install multiple ansible versions to the root (via multiple fake + # venvs) and the installation fails due to conflicts. + SETUPTOOLS_USE_DISTUTILS: stdlib - name: Copy inventory copy: diff --git a/setup.cfg b/setup.cfg index 6613a76239..207b5bbb7c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,6 +16,7 @@ classifier = Programming Language :: Python :: 3 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 [pbr] warnerrors = True diff --git a/tools/test-setup.sh b/tools/test-setup.sh index 2377635696..cc67d012b0 100755 --- a/tools/test-setup.sh +++ b/tools/test-setup.sh @@ -44,8 +44,8 @@ mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e " CREATE DATABASE openstack_citest CHARACTER SET utf8;" # setup postgres user and database -sudo -u postgres psql -c "CREATE ROLE $DB_USER WITH LOGIN SUPERUSER PASSWORD '$DB_PW';" -sudo -u postgres psql -c "CREATE DATABASE openstack_citest OWNER $DB_USER TEMPLATE template0 ENCODING 'UTF8';" +sudo -Hi -u postgres psql -c "CREATE ROLE $DB_USER WITH LOGIN SUPERUSER PASSWORD '$DB_PW';" +sudo -Hi -u postgres psql -c "CREATE DATABASE openstack_citest OWNER $DB_USER TEMPLATE template0 ENCODING 'UTF8';" LSBDISTCODENAME=$(lsb_release -cs) if [ $LSBDISTCODENAME == 'xenial' ]; then