From 47d99524c5184d281d5634a31606e5b6f3c0aca7 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Wed, 12 Apr 2023 08:48:04 +1000 Subject: [PATCH] wheel builds : move to individual releases The publish jobs build and copy the wheels to the AFS r/w partitions, but they are not released to the mirrors until a final "vos release" of their volume. Previously this happened in a final job that would not trigger if any of the wheel builds failed. This has meant thing like out-of-sync centos mirrors or arm64 node failures have stopped all wheel publishing previously. There's really no need to block other platforms publishing if one of these jobs fails. This converts the jobs to have a release job syncrhonized by a semaphore. We don't want many "vos release" processes running all at once, as it has been a source of failure with openafs before. Since releasing requires the admin key, we keep it in a separate job from the building. Change-Id: I10c307c2d46c0e5b86732943208c3167da28a694 --- playbooks/wheel/release.yaml | 45 +---------- zuul.d/jobs.yaml | 145 +++++++++++++++++++++++++++++++---- zuul.d/projects.yaml | 94 +++++++++++++++-------- 3 files changed, 194 insertions(+), 90 deletions(-) diff --git a/playbooks/wheel/release.yaml b/playbooks/wheel/release.yaml index ee8198bcd8..3b0f3120ee 100644 --- a/playbooks/wheel/release.yaml +++ b/playbooks/wheel/release.yaml @@ -2,50 +2,7 @@ roles: - role: create-afs-token - # CentOS - role: release-afs-volume - afs_volume: mirror.wheel.cent7x64 + afs_volume: '{{ wheel_volume }}' - # CentOS 8 Stream - - role: release-afs-volume - afs_volume: mirror.wheel.cent8sx64 - - role: release-afs-volume - afs_volume: mirror.wheel.cent8sa64 - - # CentOS 9 Stream - - role: release-afs-volume - afs_volume: mirror.wheel.cent9sx64 - - role: release-afs-volume - afs_volume: mirror.wheel.cent9sa64 - - # Debian - - role: release-afs-volume - afs_volume: mirror.wheel.busterx64 - - role: release-afs-volume - afs_volume: mirror.wheel.bustera64 - - role: release-afs-volume - afs_volume: mirror.wheel.deb11x64 - - role: release-afs-volume - afs_volume: mirror.wheel.deb11a64 - - # Ubuntu - - role: release-afs-volume - afs_volume: mirror.wheel.xenialx64 - - - role: release-afs-volume - afs_volume: mirror.wheel.bionicx64 - - role: release-afs-volume - afs_volume: mirror.wheel.bionica64 - - - role: release-afs-volume - afs_volume: mirror.wheel.focalx64 - - role: release-afs-volume - afs_volume: mirror.wheel.focala64 - - - role: release-afs-volume - afs_volume: mirror.wheel.jammyx64 - - role: release-afs-volume - afs_volume: mirror.wheel.jammya64 - - # Fin - role: destroy-afs-token diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 61b3f53678..73920da00d 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -853,6 +853,27 @@ run: playbooks/proposal/propose-project-config-update.yaml protected: true +# The wheel builds can finish at similar times, so to avoid many +# overlapping "vos release" processes we restrict it with a semaphore. +- semaphore: + name: wheel-release + max: 1 + +- job: + name: release-wheel-cache + abstract: true + protected: true + description: | + Release a wheel cache volume + run: + - name: playbooks/release/wheel.yaml + semaphores: wheel-release + nodeset: + nodes: [] + secrets: + - name: afs + secret: afsadmin_keytab + - job: name: publish-wheel-cache-centos-7 description: | @@ -867,6 +888,13 @@ - name: afs secret: wheel_keytab +- job: + name: release-wheel-cache-centos-7 + parent: release-wheel-cache + final: true + vars: + wheel_volume: 'mirror.wheel.cent7x64' + - job: name: publish-wheel-cache-centos-8-stream description: | @@ -881,6 +909,13 @@ - name: afs secret: wheel_keytab +- job: + name: release-wheel-cache-centos-8-stream + parent: release-wheel-cache + final: true + vars: + wheel_volume: 'mirror.wheel.cent8x64' + - job: name: publish-wheel-cache-centos-8-stream-arm64 description: | @@ -895,6 +930,13 @@ - name: afs secret: wheel_keytab +- job: + name: release-wheel-cache-centos-8-stream-arm64 + parent: release-wheel-cache + final: true + vars: + wheel_volume: 'mirror.wheel.cent8a64' + - job: name: publish-wheel-cache-centos-9-stream description: | @@ -909,6 +951,13 @@ - name: afs secret: wheel_keytab +- job: + name: release-wheel-cache-centos-9-stream + parent: release-wheel-cache + final: true + vars: + wheel_volume: 'mirror.wheel.cent9sx64' + - job: name: publish-wheel-cache-centos-9-stream-arm64 description: | @@ -923,6 +972,13 @@ - name: afs secret: wheel_keytab +- job: + name: release-wheel-cache-centos-9-stream-arm64 + parent: release-wheel-cache + final: true + vars: + wheel_volume: 'mirror.wheel.cent9sx64' + - job: name: publish-wheel-cache-debian-bullseye description: | @@ -937,6 +993,13 @@ - name: afs secret: wheel_keytab +- job: + name: release-wheel-cache-debian-bullseye + parent: release-wheel-cache + final: true + vars: + wheel_volume: 'mirror.wheel.deb11x64' + - job: name: publish-wheel-cache-debian-bullseye-arm64 description: | @@ -951,6 +1014,13 @@ - name: afs secret: wheel_keytab +- job: + name: release-wheel-cache-debian-bullseye-arm64 + parent: release-wheel-cache + final: true + vars: + wheel_volume: 'mirror.wheel.deb11a64' + - job: name: publish-wheel-cache-debian-buster description: | @@ -965,6 +1035,13 @@ - name: afs secret: wheel_keytab +- job: + name: release-wheel-cache-debian-buster + parent: release-wheel-cache + final: true + vars: + wheel_volume: 'mirror.wheel.busterx64' + - job: name: publish-wheel-cache-debian-buster-arm64 description: | @@ -979,6 +1056,13 @@ - name: afs secret: wheel_keytab +- job: + name: release-wheel-cache-debian-buster-arm64 + parent: release-wheel-cache + final: true + vars: + wheel_volume: 'mirror.wheel.bustera64' + - job: name: publish-wheel-cache-ubuntu-jammy parent: build-wheel-cache-ubuntu-jammy @@ -993,6 +1077,13 @@ - name: afs secret: wheel_keytab +- job: + name: release-wheel-cache-ubuntu-jammy + parent: release-wheel-cache + final: true + vars: + wheel_volume: 'mirror.wheel.jammyx64' + - job: name: publish-wheel-cache-ubuntu-jammy-arm64 parent: build-wheel-cache-ubuntu-jammy-arm64 @@ -1007,6 +1098,13 @@ - name: afs secret: wheel_keytab +- job: + name: release-wheel-cache-ubuntu-jammy-arm64 + parent: release-wheel-cache + final: true + vars: + wheel_volume: 'mirror.wheel.jammya64' + - job: name: publish-wheel-cache-ubuntu-focal parent: build-wheel-cache-ubuntu-focal @@ -1021,6 +1119,13 @@ - name: afs secret: wheel_keytab +- job: + name: release-wheel-cache-ubuntu-focal + parent: release-wheel-cache + final: true + vars: + wheel_volume: 'mirror.wheel.focalx64' + - job: name: publish-wheel-cache-ubuntu-focal-arm64 parent: build-wheel-cache-ubuntu-focal-arm64 @@ -1035,6 +1140,13 @@ - name: afs secret: wheel_keytab +- job: + name: release-wheel-cache-ubuntu-focal-arm64 + parent: release-wheel-cache + final: true + vars: + wheel_volume: 'mirror.wheel.focala64' + - job: name: publish-wheel-cache-ubuntu-bionic parent: build-wheel-cache-ubuntu-bionic @@ -1049,6 +1161,13 @@ - name: afs secret: wheel_keytab +- job: + name: release-wheel-cache-ubuntu-bionic + parent: release-wheel-cache + final: true + vars: + wheel_volume: 'mirror.wheel.bionicx64' + - job: name: publish-wheel-cache-ubuntu-bionic-arm64 parent: build-wheel-cache-ubuntu-bionic-arm64 @@ -1063,6 +1182,13 @@ - name: afs secret: wheel_keytab +- job: + name: release-wheel-cache-ubuntu-bionic-arm64 + parent: release-wheel-cache + final: true + vars: + wheel_volume: 'mirror.wheel.bionica64' + - job: name: publish-wheel-cache-ubuntu-xenial parent: build-wheel-cache-ubuntu-xenial @@ -1078,20 +1204,11 @@ secret: wheel_keytab - job: - name: release-wheel-cache - description: | - Release published wheels to OpenStack CI mirrors. - run: playbooks/wheel/release.yaml - nodeset: - nodes: [] - # Build and publish jobs for the wheel mirror only run against - # master. Restrict the release job to master as well to avoid zuul - # config errors due to stable branches not being able to run their - # job depenendencies. - branches: master - secrets: - - name: afs - secret: afsadmin_keytab + name: release-wheel-cache-ubuntu-xenial + parent: release-wheel-cache + final: true + vars: + wheel_volume: 'mirror.wheel.xenialx64' - job: name: check-release-approval diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 2bb934a1a4..909442c592 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -4584,39 +4584,69 @@ periodic: jobs: - publish-wheel-cache-centos-7 - - publish-wheel-cache-centos-8-stream - - publish-wheel-cache-centos-9-stream - - publish-wheel-cache-debian-bullseye - - publish-wheel-cache-debian-buster - - publish-wheel-cache-ubuntu-jammy - - publish-wheel-cache-ubuntu-focal - - publish-wheel-cache-ubuntu-bionic - - publish-wheel-cache-ubuntu-xenial - - publish-wheel-cache-centos-8-stream-arm64 - - publish-wheel-cache-centos-9-stream-arm64 - - publish-wheel-cache-debian-bullseye-arm64 - - publish-wheel-cache-debian-buster-arm64 - - publish-wheel-cache-ubuntu-jammy-arm64 - - publish-wheel-cache-ubuntu-focal-arm64 - - publish-wheel-cache-ubuntu-bionic-arm64 - - release-wheel-cache: + - release-wheel-cache-centos-7: dependencies: - - publish-wheel-cache-centos-7 - - publish-wheel-cache-centos-8-stream - - publish-wheel-cache-centos-9-stream - - publish-wheel-cache-debian-bullseye - - publish-wheel-cache-debian-buster - - publish-wheel-cache-ubuntu-jammy - - publish-wheel-cache-ubuntu-focal - - publish-wheel-cache-ubuntu-bionic - - publish-wheel-cache-ubuntu-xenial - - publish-wheel-cache-centos-8-stream-arm64 - - publish-wheel-cache-centos-9-stream-arm64 - - publish-wheel-cache-debian-bullseye-arm64 - - publish-wheel-cache-debian-buster-arm64 - - publish-wheel-cache-ubuntu-jammy-arm64 - - publish-wheel-cache-ubuntu-focal-arm64 - - publish-wheel-cache-ubuntu-bionic-arm64 + - name: publish-wheel-cache-centos-7 + - publish-wheel-cache-centos-8-stream + - release-wheel-cache-centos-8-stream: + dependencies: + - name: publish-wheel-cache-centos-8-stream + - publish-wheel-cache-centos-9-stream + - release-wheel-cache-centos-9-stream: + dependencies: + - name: publish-wheel-cache-centos-9-stream + - publish-wheel-cache-debian-bullseye + - release-wheel-cache-debian-bullseye: + dependencies: + - name: publish-wheel-cache-debian-bullseye + - publish-wheel-cache-debian-buster + - release-wheel-cache-debian-buster: + dependencies: + - name: publish-wheel-cache-debian-buster + - publish-wheel-cache-ubuntu-jammy + - release-wheel-cache-ubuntu-jammy: + dependencies: + - name: publish-wheel-cache-ubuntu-jammy + - publish-wheel-cache-ubuntu-focal + - release-wheel-cache-ubuntu-focal: + dependencies: + - name: publish-wheel-cache-ubuntu-focal + - publish-wheel-cache-ubuntu-bionic + - release-wheel-cache-ubuntu-bionic: + dependencies: + - name: publish-wheel-cache-ubuntu-bionic + - publish-wheel-cache-ubuntu-xenial + - release-wheel-cache-ubuntu-xenial: + dependencies: + - name: publish-wheel-cache-ubuntu-xenial + - publish-wheel-cache-centos-8-stream-arm64 + - release-wheel-cache-centos-8-stream-arm64: + dependencies: + - name: publish-wheel-cache-centos-8-stream-arm64 + - publish-wheel-cache-centos-9-stream-arm64 + - release-wheel-cache-centos-9-stream-arm64: + dependencies: + - name: publish-wheel-cache-centos-9-stream-arm64 + - publish-wheel-cache-debian-bullseye-arm64 + - release-wheel-cache-debian-bullseye-arm64: + dependencies: + - name: publish-wheel-cache-debian-bullseye-arm64 + - publish-wheel-cache-debian-buster-arm64 + - release-wheel-cache-debian-buster-arm64: + dependencies: + - name: publish-wheel-cache-debian-buster-arm64 + - publish-wheel-cache-ubuntu-jammy-arm64 + - release-wheel-cache-ubuntu-jammy-arm64: + dependencies: + - name: publish-wheel-cache-ubuntu-jammy-arm64 + - publish-wheel-cache-ubuntu-focal-arm64 + - release-wheel-cache-ubuntu-focal-arm64: + dependencies: + - name: publish-wheel-cache-ubuntu-focal-arm64 + - publish-wheel-cache-ubuntu-bionic-arm64 + - release-wheel-cache-ubuntu-bionic-arm64: + dependencies: + - name: publish-wheel-cache-ubuntu-bionic-arm64 periodic-weekly: jobs: - propose-updates: