From 6c001b528ce8c04ec03b51b9a8a358bc4ba33c77 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Thu, 28 May 2020 13:28:09 +1000 Subject: [PATCH] Move wheel build role/job outside project-config We can not speculatively test changes to the wheel build jobs while it is project config. These jobs are in the gate for the requirements project, which run them when bindep.txt changes. https://review.opendev.org/731394 moved the build-wheel-mirror-base job/playbook/role into openstack-zuul-jobs, renamed as build-wheel-cache-* to a) distinguish the name and b) make it a little clearer that we're building the wheels that goes on the mirror, rather than passing through to mirroring something existing. This updates the job names in project-config to reference these new jobs. Note the publishing steps are kept here along with the AFS secret. Depends-On: https://review.opendev.org/732085 Depends-On: https://review.opendev.org/732087 Depends-On: https://review.opendev.org/732083 Depends-On: https://review.opendev.org/732084 Depends-On: https://review.opendev.org/732086 Change-Id: Iac3a906803177e8365f4cfb611800b5ccaed4a6e --- playbooks/wheel/build.yaml | 10 -- playbooks/wheel/post-base.yaml | 14 -- playbooks/wheel/pre.yaml | 26 --- roles/build-wheels/README.rst | 1 - roles/build-wheels/files/wheel-build.sh | 90 ---------- roles/build-wheels/tasks/main.yaml | 17 -- zuul.d/jobs.yaml | 224 +++--------------------- zuul.d/projects.yaml | 46 ++--- 8 files changed, 48 insertions(+), 380 deletions(-) delete mode 100644 playbooks/wheel/build.yaml delete mode 100644 playbooks/wheel/post-base.yaml delete mode 100644 playbooks/wheel/pre.yaml delete mode 100644 roles/build-wheels/README.rst delete mode 100755 roles/build-wheels/files/wheel-build.sh delete mode 100644 roles/build-wheels/tasks/main.yaml diff --git a/playbooks/wheel/build.yaml b/playbooks/wheel/build.yaml deleted file mode 100644 index 2e20784b80..0000000000 --- a/playbooks/wheel/build.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- hosts: all - strategy: free - - pre_tasks: - - name: Set fact for wheel dir - set_fact: - wheel_dir: /opt/wheel/workspace - - roles: - - build-wheels diff --git a/playbooks/wheel/post-base.yaml b/playbooks/wheel/post-base.yaml deleted file mode 100644 index 36dd3c0993..0000000000 --- a/playbooks/wheel/post-base.yaml +++ /dev/null @@ -1,14 +0,0 @@ -- hosts: all - tasks: - - name: Ensure build logs directory exists - file: - path: "{{ zuul.executor.log_root }}/{{ wheel_python }}" - state: directory - delegate_to: localhost - - - name: Collect build logs - synchronize: - dest: "{{ zuul.executor.log_root }}/{{ wheel_python }}" - mode: pull - src: ~/logs/ - verify_host: true diff --git a/playbooks/wheel/pre.yaml b/playbooks/wheel/pre.yaml deleted file mode 100644 index 3367da27c0..0000000000 --- a/playbooks/wheel/pre.yaml +++ /dev/null @@ -1,26 +0,0 @@ -- hosts: all - roles: - - bindep - - pre_tasks: - # NOTE(pabelanger): Manually install EPEL dependencies as not to - # keep EPEL enabled. - - name: Manually install EPEL dependencies - become: yes - yum: - name: "{{ item }}" - enablerepo: epel - state: present - loop: - - parallel - when: ansible_distribution == "CentOS" - - # NOTE(pabelanger): We need to install extra libraries, such as - # liberasurecode-devel - # which are only part of the RDO repositories. - - name: Add centos-release-openstack-queens support - yum: - name: centos-release-openstack-queens - state: present - become: yes - when: ansible_distribution == "CentOS" diff --git a/roles/build-wheels/README.rst b/roles/build-wheels/README.rst deleted file mode 100644 index 6e4c38c390..0000000000 --- a/roles/build-wheels/README.rst +++ /dev/null @@ -1 +0,0 @@ -Run the build scripts on the wheel mirror builders diff --git a/roles/build-wheels/files/wheel-build.sh b/roles/build-wheels/files/wheel-build.sh deleted file mode 100755 index eb1c833da0..0000000000 --- a/roles/build-wheels/files/wheel-build.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash -xe - -# Working variables -WHEELHOUSE_DIR=$1 -WORKING_DIR=$(pwd)/src/opendev.org/openstack/requirements -PYTHON_VERSION=$2 -LOGS=$(pwd)/logs - -FAIL_LOG=${LOGS}/failed.txt - -# preclean logs -mkdir -p ${LOGS} -rm -rf ${LOGS}/* - -# Extract and iterate over all the branch names. -BRANCHES=`git --git-dir=$WORKING_DIR/.git branch -a | grep '^ stable' | \ - grep -Ev '(newton)'` -for BRANCH in master $BRANCHES; do - git --git-dir=$WORKING_DIR/.git show $BRANCH:upper-constraints.txt \ - 2>/dev/null > /tmp/upper-constraints.txt || true - - # setup the building virtualenv. We want to freshen this for each - # branch. - rm -rf build_env - virtualenv -p $PYTHON_VERSION build_env - - build_env/bin/pip install --upgrade pip - - # SHORT_BRANCH is just "master","newton","kilo" etc. because this - # keeps the output log hierarchy much simpler. - SHORT_BRANCH=${BRANCH##origin/} - SHORT_BRANCH=${SHORT_BRANCH##stable/} - - # Failed parallel jobs don't fail the whole job, we just report - # the issues for investigation. - set +e - - # This runs all the jobs under "parallel". The stdout, stderr and - # exit status for each pip invocation will be captured into files - # kept in ${LOGS}/build/${SHORT_BRANCH}/1/[package]. The --joblog - # file keeps an overview of all run jobs, which we can probe to - # find failed jobs. - cat /tmp/upper-constraints.txt | \ - parallel --files --progress --joblog ${LOGS}/$SHORT_BRANCH-job.log \ - --results ${LOGS}/build/$SHORT_BRANCH \ - build_env/bin/pip --verbose --exists-action=i wheel \ - -c /tmp/upper-constraints.txt \ - -w $WHEELHOUSE_DIR {} - set -e - - # Column $7 is the exit status of the job, $NF is the last - # argument to pip, which is our package. - FAILED=$(awk -e '$7!=0 {print $NF}' ${LOGS}/$SHORT_BRANCH-job.log) - if [ -n "${FAILED}" ]; then - echo "*** FAILED BUILDS FOR BRANCH ${BRANCH}" >> ${FAIL_LOG} - echo "${FAILED}" >> ${FAIL_LOG} - echo -e "***\n\n" >> ${FAIL_LOG} - fi -done - -# Parse stdout from all the build logs to build up a unique list of all -# wheels downloaded from PyPI and delete them from the wheelhouse, since -# this is only meant to provide built wheels which are absent from PyPI. -find ${LOGS}/build/ -name stdout -exec grep 'Downloading' {} \; \ - | sed -n 's,.*Downloading.*[ /]\([^ /]*\.whl\)\([ #].*\|$\),\1,p' \ - | sort -u > ${LOGS}/remove-wheels.txt -pushd ${WHEELHOUSE_DIR} -cat ${LOGS}/remove-wheels.txt | xargs rm -popd - -if [ -f ${FAIL_LOG} ]; then - cat ${FAIL_LOG} -fi - -# XXX This does make a lot of log files; about 80mb after compression. -# In theory we could correlate just the failed logs and keep those -# from the failure logs above. This is currently (2017-01) left as an -# exercise for when the job is stable :) bz2 gave about 20% -# improvement over gzip in testing. -pushd ${LOGS} -tar zcvf build-logs.tar.gz ./build -rm -rf ./build -popd - -# Set the final exit status to 1 if remove-wheels.txt is empty so the -# job will fail. -if [ ! -s ${LOGS}/remove-wheels.txt ]; then - echo "*** EMPTY WHEEL REMOVAL LIST: THIS SHOULD NOT HAPPEN ***" - exit 1 -fi diff --git a/roles/build-wheels/tasks/main.yaml b/roles/build-wheels/tasks/main.yaml deleted file mode 100644 index f332011d64..0000000000 --- a/roles/build-wheels/tasks/main.yaml +++ /dev/null @@ -1,17 +0,0 @@ -- name: Ensure wheel dir exists - file: - group: zuul - owner: zuul - path: "{{ wheel_dir }}" - state: directory - become: yes - -- name: Prevent using existing wheel mirror - become: yes - lineinfile: - path: /etc/pip.conf - state: absent - regexp: '^extra-index-url.*$' - -- name: Build the wheel mirror - script: wheel-build.sh {{ wheel_dir }} {{ wheel_python }} diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 640867f304..4390cd21b2 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -776,7 +776,8 @@ - job: name: openstack-upload-github-mirror parent: opendev-upload-git-mirror - description: Mirrors official projects to github's openstack org + description: | + Mirrors official projects to github's openstack org final: true protected: true secrets: @@ -817,31 +818,10 @@ protected: true - job: - # NOTE(ianw) : We are in the process of removing the - # "wheel-mirror" jobs in project-config as we have moved them into - # openstack-zuul-jobs. This is just until requirements removes - # all references to the old jobs. - name: build-wheel-mirror-base - parent: build-wheel-cache-base - -- job: - name: build-wheel-mirror-centos-7 - parent: build-wheel-mirror-base - description: | - Build CentOS 7 wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-centos-7-python2 - label: centos-7 - host-vars: - wheel-mirror-centos-7-python2: - wheel_python: python2 - -- job: - name: publish-wheel-mirror-centos-7 + name: publish-wheel-cache-centos-7 description: | Publish CentOS 7 wheels for OpenStack CI mirrors. - parent: build-wheel-mirror-centos-7 + parent: build-wheel-cache-centos-7 pre-run: playbooks/openafs-client/setup.yaml post-run: playbooks/publish/wheel-mirror.yaml allowed-projects: @@ -852,23 +832,10 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-centos-8 - parent: build-wheel-mirror-base - description: | - Build CentOS 8 wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-centos-8-python3 - label: centos-8 - host-vars: - wheel-mirror-centos-8-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-centos-8 + name: publish-wheel-cache-centos-8 description: | Publish CentOS 8 wheels for OpenStack CI mirrors. - parent: build-wheel-mirror-centos-8 + parent: build-wheel-cache-centos-8 pre-run: playbooks/openafs-client/setup.yaml post-run: playbooks/publish/wheel-mirror.yaml allowed-projects: @@ -879,23 +846,10 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-centos-8-arm64 - parent: build-wheel-mirror-base - description: | - Build CentOS 8 wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-centos-8-arm64-python3 - label: centos-8-arm64 - host-vars: - wheel-mirror-centos-8-arm64-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-centos-8-arm64 + name: publish-wheel-cache-centos-8-arm64 description: | Publish CentOS 8 wheels for OpenStack CI mirrors. - parent: build-wheel-mirror-centos-8-arm64 + parent: build-wheel-cache-centos-8-arm64 pre-run: playbooks/openafs-client/setup.yaml post-run: playbooks/publish/wheel-mirror.yaml allowed-projects: @@ -906,27 +860,10 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-debian-buster - parent: build-wheel-mirror-base - description: | - Build Debian buster wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-debian-buster-python2 - label: debian-buster - - name: wheel-mirror-debian-buster-python3 - label: debian-buster - host-vars: - wheel-mirror-debian-buster-python2: - wheel_python: python2 - wheel-mirror-debian-buster-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-debian-buster + name: publish-wheel-cache-debian-buster description: | Publish Debian buster wheels for OpenStack CI mirrors. - parent: build-wheel-mirror-debian-buster + parent: build-wheel-cache-debian-buster pre-run: playbooks/openafs-client/setup.yaml post-run: playbooks/publish/wheel-mirror.yaml allowed-projects: @@ -937,27 +874,10 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-debian-buster-arm64 - parent: build-wheel-mirror-base - description: | - Build Debian buster wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-debian-buster-arm64-python2 - label: debian-buster-arm64 - - name: wheel-mirror-debian-buster-arm64-python3 - label: debian-buster-arm64 - host-vars: - wheel-mirror-debian-buster-arm64-python2: - wheel_python: python2 - wheel-mirror-debian-buster-arm64-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-debian-buster-arm64 + name: publish-wheel-cache-debian-buster-arm64 description: | Publish Debian buster wheels for OpenStack CI mirrors. - parent: build-wheel-mirror-debian-buster-arm64 + parent: build-wheel-cache-debian-buster-arm64 pre-run: playbooks/openafs-client/setup.yaml post-run: playbooks/publish/wheel-mirror.yaml allowed-projects: @@ -968,21 +888,8 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-ubuntu-focal - parent: build-wheel-mirror-base - description: | - Build Ubuntu Focal wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-ubuntu-focal-python3 - label: ubuntu-focal - host-vars: - wheel-mirror-ubuntu-focal-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-ubuntu-focal - parent: build-wheel-mirror-ubuntu-focal + name: publish-wheel-cache-ubuntu-focal + parent: build-wheel-cache-ubuntu-focal description: | Publish Ubuntu Focal wheels for OpenStack CI mirrors. pre-run: playbooks/openafs-client/setup.yaml @@ -995,21 +902,8 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-ubuntu-focal-arm64 - parent: build-wheel-mirror-base - description: | - Build Ubuntu Focal wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-ubuntu-focal-arm64-python3 - label: ubuntu-focal-arm64 - host-vars: - wheel-mirror-ubuntu-focal-arm64-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-ubuntu-focal-arm64 - parent: build-wheel-mirror-ubuntu-focal-arm64 + name: publish-wheel-cache-ubuntu-focal-arm64 + parent: build-wheel-cache-ubuntu-focal-arm64 description: | Publish Ubuntu Focal wheels for OpenStack CI mirrors. pre-run: playbooks/openafs-client/setup.yaml @@ -1022,25 +916,8 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-ubuntu-bionic - parent: build-wheel-mirror-base - description: | - Build Ubuntu Bionic wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-ubuntu-bionic-python2 - label: ubuntu-bionic - - name: wheel-mirror-ubuntu-bionic-python3 - label: ubuntu-bionic - host-vars: - wheel-mirror-ubuntu-bionic-python2: - wheel_python: python2 - wheel-mirror-ubuntu-bionic-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-ubuntu-bionic - parent: build-wheel-mirror-ubuntu-bionic + name: publish-wheel-cache-ubuntu-bionic + parent: build-wheel-cache-ubuntu-bionic description: | Publish Ubuntu Bionic wheels for OpenStack CI mirrors. pre-run: playbooks/openafs-client/setup.yaml @@ -1053,25 +930,8 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-ubuntu-bionic-arm64 - parent: build-wheel-mirror-base - description: | - Build Ubuntu Bionic wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-ubuntu-bionic-arm64-python2 - label: ubuntu-bionic-arm64 - - name: wheel-mirror-ubuntu-bionic-arm64-python3 - label: ubuntu-bionic-arm64 - host-vars: - wheel-mirror-ubuntu-bionic-arm64-python2: - wheel_python: python2 - wheel-mirror-ubuntu-bionic-arm64-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-ubuntu-bionic-arm64 - parent: build-wheel-mirror-ubuntu-bionic-arm64 + name: publish-wheel-cache-ubuntu-bionic-arm64 + parent: build-wheel-cache-ubuntu-bionic-arm64 description: | Publish Ubuntu Bionic wheels for OpenStack CI mirrors. pre-run: playbooks/openafs-client/setup.yaml @@ -1084,25 +944,8 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-ubuntu-xenial - parent: build-wheel-mirror-base - description: | - Build Ubuntu Xenial wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-ubuntu-xenial-python2 - label: ubuntu-xenial - - name: wheel-mirror-ubuntu-xenial-python3 - label: ubuntu-xenial - host-vars: - wheel-mirror-ubuntu-xenial-python2: - wheel_python: python2 - wheel-mirror-ubuntu-xenial-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-ubuntu-xenial - parent: build-wheel-mirror-ubuntu-xenial + name: publish-wheel-cache-ubuntu-xenial + parent: build-wheel-cache-ubuntu-xenial description: | Publish Ubuntu Xenial wheels for OpenStack CI mirrors. pre-run: playbooks/openafs-client/setup.yaml @@ -1115,25 +958,8 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-ubuntu-xenial-arm64 - parent: build-wheel-mirror-base - description: | - Build Ubuntu Xenial ARM64 wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-ubuntu-xenial-arm64-python2 - label: ubuntu-xenial-arm64 - - name: wheel-mirror-ubuntu-xenial-arm64-python3 - label: ubuntu-xenial-arm64 - host-vars: - wheel-mirror-ubuntu-xenial-arm64-python2: - wheel_python: python2 - wheel-mirror-ubuntu-xenial-arm64-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-ubuntu-xenial-arm64 - parent: build-wheel-mirror-ubuntu-xenial-arm64 + name: publish-wheel-cache-ubuntu-xenial-arm64 + parent: build-wheel-cache-ubuntu-xenial-arm64 description: | Publish Ubuntu Xenial ARM64 wheels for OpenStack CI mirrors. pre-run: playbooks/openafs-client/setup.yaml @@ -1146,7 +972,7 @@ secret: wheel_keytab - job: - name: release-wheel-mirror + name: release-wheel-cache description: | Release published wheels to OpenStack CI mirrors. run: playbooks/wheel/release.yaml diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 02ee8c1a05..5cfd9c44b2 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -5413,30 +5413,30 @@ branches: master periodic: jobs: - - publish-wheel-mirror-centos-7 - - publish-wheel-mirror-centos-8 - - publish-wheel-mirror-debian-buster - - publish-wheel-mirror-ubuntu-focal - - publish-wheel-mirror-ubuntu-bionic - - publish-wheel-mirror-ubuntu-xenial - - publish-wheel-mirror-centos-8-arm64 - - publish-wheel-mirror-debian-buster-arm64 - - publish-wheel-mirror-ubuntu-focal-arm64 - - publish-wheel-mirror-ubuntu-bionic-arm64 - - publish-wheel-mirror-ubuntu-xenial-arm64 - - release-wheel-mirror: + - publish-wheel-cache-centos-7 + - publish-wheel-cache-centos-8 + - publish-wheel-cache-debian-buster + - publish-wheel-cache-ubuntu-focal + - publish-wheel-cache-ubuntu-bionic + - publish-wheel-cache-ubuntu-xenial + - publish-wheel-cache-centos-8-arm64 + - publish-wheel-cache-debian-buster-arm64 + - publish-wheel-cache-ubuntu-focal-arm64 + - publish-wheel-cache-ubuntu-bionic-arm64 + - publish-wheel-cache-ubuntu-xenial-arm64 + - release-wheel-cache: dependencies: - - publish-wheel-mirror-centos-7 - - publish-wheel-mirror-centos-8 - - publish-wheel-mirror-debian-buster - - publish-wheel-mirror-ubuntu-focal - - publish-wheel-mirror-ubuntu-bionic - - publish-wheel-mirror-ubuntu-xenial - - publish-wheel-mirror-centos-8-arm64 - - publish-wheel-mirror-debian-buster-arm64 - - publish-wheel-mirror-ubuntu-focal-arm64 - - publish-wheel-mirror-ubuntu-bionic-arm64 - - publish-wheel-mirror-ubuntu-xenial-arm64 + - publish-wheel-cache-centos-7 + - publish-wheel-cache-centos-8 + - publish-wheel-cache-debian-buster + - publish-wheel-cache-ubuntu-focal + - publish-wheel-cache-ubuntu-bionic + - publish-wheel-cache-ubuntu-xenial + - publish-wheel-cache-centos-8-arm64 + - publish-wheel-cache-debian-buster-arm64 + - publish-wheel-cache-ubuntu-focal-arm64 + - publish-wheel-cache-ubuntu-bionic-arm64 + - publish-wheel-cache-ubuntu-xenial-arm64 - propose-updates: nodeset: ubuntu-bionic branches: master