diff --git a/playbooks/wheel/build.yaml b/playbooks/wheel/build.yaml deleted file mode 100644 index 2e20784b80..0000000000 --- a/playbooks/wheel/build.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- hosts: all - strategy: free - - pre_tasks: - - name: Set fact for wheel dir - set_fact: - wheel_dir: /opt/wheel/workspace - - roles: - - build-wheels diff --git a/playbooks/wheel/post-base.yaml b/playbooks/wheel/post-base.yaml deleted file mode 100644 index 36dd3c0993..0000000000 --- a/playbooks/wheel/post-base.yaml +++ /dev/null @@ -1,14 +0,0 @@ -- hosts: all - tasks: - - name: Ensure build logs directory exists - file: - path: "{{ zuul.executor.log_root }}/{{ wheel_python }}" - state: directory - delegate_to: localhost - - - name: Collect build logs - synchronize: - dest: "{{ zuul.executor.log_root }}/{{ wheel_python }}" - mode: pull - src: ~/logs/ - verify_host: true diff --git a/playbooks/wheel/pre.yaml b/playbooks/wheel/pre.yaml deleted file mode 100644 index 3367da27c0..0000000000 --- a/playbooks/wheel/pre.yaml +++ /dev/null @@ -1,26 +0,0 @@ -- hosts: all - roles: - - bindep - - pre_tasks: - # NOTE(pabelanger): Manually install EPEL dependencies as not to - # keep EPEL enabled. - - name: Manually install EPEL dependencies - become: yes - yum: - name: "{{ item }}" - enablerepo: epel - state: present - loop: - - parallel - when: ansible_distribution == "CentOS" - - # NOTE(pabelanger): We need to install extra libraries, such as - # liberasurecode-devel - # which are only part of the RDO repositories. - - name: Add centos-release-openstack-queens support - yum: - name: centos-release-openstack-queens - state: present - become: yes - when: ansible_distribution == "CentOS" diff --git a/roles/build-wheels/README.rst b/roles/build-wheels/README.rst deleted file mode 100644 index 6e4c38c390..0000000000 --- a/roles/build-wheels/README.rst +++ /dev/null @@ -1 +0,0 @@ -Run the build scripts on the wheel mirror builders diff --git a/roles/build-wheels/files/wheel-build.sh b/roles/build-wheels/files/wheel-build.sh deleted file mode 100755 index eb1c833da0..0000000000 --- a/roles/build-wheels/files/wheel-build.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash -xe - -# Working variables -WHEELHOUSE_DIR=$1 -WORKING_DIR=$(pwd)/src/opendev.org/openstack/requirements -PYTHON_VERSION=$2 -LOGS=$(pwd)/logs - -FAIL_LOG=${LOGS}/failed.txt - -# preclean logs -mkdir -p ${LOGS} -rm -rf ${LOGS}/* - -# Extract and iterate over all the branch names. -BRANCHES=`git --git-dir=$WORKING_DIR/.git branch -a | grep '^ stable' | \ - grep -Ev '(newton)'` -for BRANCH in master $BRANCHES; do - git --git-dir=$WORKING_DIR/.git show $BRANCH:upper-constraints.txt \ - 2>/dev/null > /tmp/upper-constraints.txt || true - - # setup the building virtualenv. We want to freshen this for each - # branch. - rm -rf build_env - virtualenv -p $PYTHON_VERSION build_env - - build_env/bin/pip install --upgrade pip - - # SHORT_BRANCH is just "master","newton","kilo" etc. because this - # keeps the output log hierarchy much simpler. - SHORT_BRANCH=${BRANCH##origin/} - SHORT_BRANCH=${SHORT_BRANCH##stable/} - - # Failed parallel jobs don't fail the whole job, we just report - # the issues for investigation. - set +e - - # This runs all the jobs under "parallel". The stdout, stderr and - # exit status for each pip invocation will be captured into files - # kept in ${LOGS}/build/${SHORT_BRANCH}/1/[package]. The --joblog - # file keeps an overview of all run jobs, which we can probe to - # find failed jobs. - cat /tmp/upper-constraints.txt | \ - parallel --files --progress --joblog ${LOGS}/$SHORT_BRANCH-job.log \ - --results ${LOGS}/build/$SHORT_BRANCH \ - build_env/bin/pip --verbose --exists-action=i wheel \ - -c /tmp/upper-constraints.txt \ - -w $WHEELHOUSE_DIR {} - set -e - - # Column $7 is the exit status of the job, $NF is the last - # argument to pip, which is our package. - FAILED=$(awk -e '$7!=0 {print $NF}' ${LOGS}/$SHORT_BRANCH-job.log) - if [ -n "${FAILED}" ]; then - echo "*** FAILED BUILDS FOR BRANCH ${BRANCH}" >> ${FAIL_LOG} - echo "${FAILED}" >> ${FAIL_LOG} - echo -e "***\n\n" >> ${FAIL_LOG} - fi -done - -# Parse stdout from all the build logs to build up a unique list of all -# wheels downloaded from PyPI and delete them from the wheelhouse, since -# this is only meant to provide built wheels which are absent from PyPI. -find ${LOGS}/build/ -name stdout -exec grep 'Downloading' {} \; \ - | sed -n 's,.*Downloading.*[ /]\([^ /]*\.whl\)\([ #].*\|$\),\1,p' \ - | sort -u > ${LOGS}/remove-wheels.txt -pushd ${WHEELHOUSE_DIR} -cat ${LOGS}/remove-wheels.txt | xargs rm -popd - -if [ -f ${FAIL_LOG} ]; then - cat ${FAIL_LOG} -fi - -# XXX This does make a lot of log files; about 80mb after compression. -# In theory we could correlate just the failed logs and keep those -# from the failure logs above. This is currently (2017-01) left as an -# exercise for when the job is stable :) bz2 gave about 20% -# improvement over gzip in testing. -pushd ${LOGS} -tar zcvf build-logs.tar.gz ./build -rm -rf ./build -popd - -# Set the final exit status to 1 if remove-wheels.txt is empty so the -# job will fail. -if [ ! -s ${LOGS}/remove-wheels.txt ]; then - echo "*** EMPTY WHEEL REMOVAL LIST: THIS SHOULD NOT HAPPEN ***" - exit 1 -fi diff --git a/roles/build-wheels/tasks/main.yaml b/roles/build-wheels/tasks/main.yaml deleted file mode 100644 index f332011d64..0000000000 --- a/roles/build-wheels/tasks/main.yaml +++ /dev/null @@ -1,17 +0,0 @@ -- name: Ensure wheel dir exists - file: - group: zuul - owner: zuul - path: "{{ wheel_dir }}" - state: directory - become: yes - -- name: Prevent using existing wheel mirror - become: yes - lineinfile: - path: /etc/pip.conf - state: absent - regexp: '^extra-index-url.*$' - -- name: Build the wheel mirror - script: wheel-build.sh {{ wheel_dir }} {{ wheel_python }} diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 640867f304..4390cd21b2 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -776,7 +776,8 @@ - job: name: openstack-upload-github-mirror parent: opendev-upload-git-mirror - description: Mirrors official projects to github's openstack org + description: | + Mirrors official projects to github's openstack org final: true protected: true secrets: @@ -817,31 +818,10 @@ protected: true - job: - # NOTE(ianw) : We are in the process of removing the - # "wheel-mirror" jobs in project-config as we have moved them into - # openstack-zuul-jobs. This is just until requirements removes - # all references to the old jobs. - name: build-wheel-mirror-base - parent: build-wheel-cache-base - -- job: - name: build-wheel-mirror-centos-7 - parent: build-wheel-mirror-base - description: | - Build CentOS 7 wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-centos-7-python2 - label: centos-7 - host-vars: - wheel-mirror-centos-7-python2: - wheel_python: python2 - -- job: - name: publish-wheel-mirror-centos-7 + name: publish-wheel-cache-centos-7 description: | Publish CentOS 7 wheels for OpenStack CI mirrors. - parent: build-wheel-mirror-centos-7 + parent: build-wheel-cache-centos-7 pre-run: playbooks/openafs-client/setup.yaml post-run: playbooks/publish/wheel-mirror.yaml allowed-projects: @@ -852,23 +832,10 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-centos-8 - parent: build-wheel-mirror-base - description: | - Build CentOS 8 wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-centos-8-python3 - label: centos-8 - host-vars: - wheel-mirror-centos-8-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-centos-8 + name: publish-wheel-cache-centos-8 description: | Publish CentOS 8 wheels for OpenStack CI mirrors. - parent: build-wheel-mirror-centos-8 + parent: build-wheel-cache-centos-8 pre-run: playbooks/openafs-client/setup.yaml post-run: playbooks/publish/wheel-mirror.yaml allowed-projects: @@ -879,23 +846,10 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-centos-8-arm64 - parent: build-wheel-mirror-base - description: | - Build CentOS 8 wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-centos-8-arm64-python3 - label: centos-8-arm64 - host-vars: - wheel-mirror-centos-8-arm64-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-centos-8-arm64 + name: publish-wheel-cache-centos-8-arm64 description: | Publish CentOS 8 wheels for OpenStack CI mirrors. - parent: build-wheel-mirror-centos-8-arm64 + parent: build-wheel-cache-centos-8-arm64 pre-run: playbooks/openafs-client/setup.yaml post-run: playbooks/publish/wheel-mirror.yaml allowed-projects: @@ -906,27 +860,10 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-debian-buster - parent: build-wheel-mirror-base - description: | - Build Debian buster wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-debian-buster-python2 - label: debian-buster - - name: wheel-mirror-debian-buster-python3 - label: debian-buster - host-vars: - wheel-mirror-debian-buster-python2: - wheel_python: python2 - wheel-mirror-debian-buster-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-debian-buster + name: publish-wheel-cache-debian-buster description: | Publish Debian buster wheels for OpenStack CI mirrors. - parent: build-wheel-mirror-debian-buster + parent: build-wheel-cache-debian-buster pre-run: playbooks/openafs-client/setup.yaml post-run: playbooks/publish/wheel-mirror.yaml allowed-projects: @@ -937,27 +874,10 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-debian-buster-arm64 - parent: build-wheel-mirror-base - description: | - Build Debian buster wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-debian-buster-arm64-python2 - label: debian-buster-arm64 - - name: wheel-mirror-debian-buster-arm64-python3 - label: debian-buster-arm64 - host-vars: - wheel-mirror-debian-buster-arm64-python2: - wheel_python: python2 - wheel-mirror-debian-buster-arm64-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-debian-buster-arm64 + name: publish-wheel-cache-debian-buster-arm64 description: | Publish Debian buster wheels for OpenStack CI mirrors. - parent: build-wheel-mirror-debian-buster-arm64 + parent: build-wheel-cache-debian-buster-arm64 pre-run: playbooks/openafs-client/setup.yaml post-run: playbooks/publish/wheel-mirror.yaml allowed-projects: @@ -968,21 +888,8 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-ubuntu-focal - parent: build-wheel-mirror-base - description: | - Build Ubuntu Focal wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-ubuntu-focal-python3 - label: ubuntu-focal - host-vars: - wheel-mirror-ubuntu-focal-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-ubuntu-focal - parent: build-wheel-mirror-ubuntu-focal + name: publish-wheel-cache-ubuntu-focal + parent: build-wheel-cache-ubuntu-focal description: | Publish Ubuntu Focal wheels for OpenStack CI mirrors. pre-run: playbooks/openafs-client/setup.yaml @@ -995,21 +902,8 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-ubuntu-focal-arm64 - parent: build-wheel-mirror-base - description: | - Build Ubuntu Focal wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-ubuntu-focal-arm64-python3 - label: ubuntu-focal-arm64 - host-vars: - wheel-mirror-ubuntu-focal-arm64-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-ubuntu-focal-arm64 - parent: build-wheel-mirror-ubuntu-focal-arm64 + name: publish-wheel-cache-ubuntu-focal-arm64 + parent: build-wheel-cache-ubuntu-focal-arm64 description: | Publish Ubuntu Focal wheels for OpenStack CI mirrors. pre-run: playbooks/openafs-client/setup.yaml @@ -1022,25 +916,8 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-ubuntu-bionic - parent: build-wheel-mirror-base - description: | - Build Ubuntu Bionic wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-ubuntu-bionic-python2 - label: ubuntu-bionic - - name: wheel-mirror-ubuntu-bionic-python3 - label: ubuntu-bionic - host-vars: - wheel-mirror-ubuntu-bionic-python2: - wheel_python: python2 - wheel-mirror-ubuntu-bionic-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-ubuntu-bionic - parent: build-wheel-mirror-ubuntu-bionic + name: publish-wheel-cache-ubuntu-bionic + parent: build-wheel-cache-ubuntu-bionic description: | Publish Ubuntu Bionic wheels for OpenStack CI mirrors. pre-run: playbooks/openafs-client/setup.yaml @@ -1053,25 +930,8 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-ubuntu-bionic-arm64 - parent: build-wheel-mirror-base - description: | - Build Ubuntu Bionic wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-ubuntu-bionic-arm64-python2 - label: ubuntu-bionic-arm64 - - name: wheel-mirror-ubuntu-bionic-arm64-python3 - label: ubuntu-bionic-arm64 - host-vars: - wheel-mirror-ubuntu-bionic-arm64-python2: - wheel_python: python2 - wheel-mirror-ubuntu-bionic-arm64-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-ubuntu-bionic-arm64 - parent: build-wheel-mirror-ubuntu-bionic-arm64 + name: publish-wheel-cache-ubuntu-bionic-arm64 + parent: build-wheel-cache-ubuntu-bionic-arm64 description: | Publish Ubuntu Bionic wheels for OpenStack CI mirrors. pre-run: playbooks/openafs-client/setup.yaml @@ -1084,25 +944,8 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-ubuntu-xenial - parent: build-wheel-mirror-base - description: | - Build Ubuntu Xenial wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-ubuntu-xenial-python2 - label: ubuntu-xenial - - name: wheel-mirror-ubuntu-xenial-python3 - label: ubuntu-xenial - host-vars: - wheel-mirror-ubuntu-xenial-python2: - wheel_python: python2 - wheel-mirror-ubuntu-xenial-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-ubuntu-xenial - parent: build-wheel-mirror-ubuntu-xenial + name: publish-wheel-cache-ubuntu-xenial + parent: build-wheel-cache-ubuntu-xenial description: | Publish Ubuntu Xenial wheels for OpenStack CI mirrors. pre-run: playbooks/openafs-client/setup.yaml @@ -1115,25 +958,8 @@ secret: wheel_keytab - job: - name: build-wheel-mirror-ubuntu-xenial-arm64 - parent: build-wheel-mirror-base - description: | - Build Ubuntu Xenial ARM64 wheels for OpenStack CI mirrors. - nodeset: - nodes: - - name: wheel-mirror-ubuntu-xenial-arm64-python2 - label: ubuntu-xenial-arm64 - - name: wheel-mirror-ubuntu-xenial-arm64-python3 - label: ubuntu-xenial-arm64 - host-vars: - wheel-mirror-ubuntu-xenial-arm64-python2: - wheel_python: python2 - wheel-mirror-ubuntu-xenial-arm64-python3: - wheel_python: python3 - -- job: - name: publish-wheel-mirror-ubuntu-xenial-arm64 - parent: build-wheel-mirror-ubuntu-xenial-arm64 + name: publish-wheel-cache-ubuntu-xenial-arm64 + parent: build-wheel-cache-ubuntu-xenial-arm64 description: | Publish Ubuntu Xenial ARM64 wheels for OpenStack CI mirrors. pre-run: playbooks/openafs-client/setup.yaml @@ -1146,7 +972,7 @@ secret: wheel_keytab - job: - name: release-wheel-mirror + name: release-wheel-cache description: | Release published wheels to OpenStack CI mirrors. run: playbooks/wheel/release.yaml diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 02ee8c1a05..5cfd9c44b2 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -5413,30 +5413,30 @@ branches: master periodic: jobs: - - publish-wheel-mirror-centos-7 - - publish-wheel-mirror-centos-8 - - publish-wheel-mirror-debian-buster - - publish-wheel-mirror-ubuntu-focal - - publish-wheel-mirror-ubuntu-bionic - - publish-wheel-mirror-ubuntu-xenial - - publish-wheel-mirror-centos-8-arm64 - - publish-wheel-mirror-debian-buster-arm64 - - publish-wheel-mirror-ubuntu-focal-arm64 - - publish-wheel-mirror-ubuntu-bionic-arm64 - - publish-wheel-mirror-ubuntu-xenial-arm64 - - release-wheel-mirror: + - publish-wheel-cache-centos-7 + - publish-wheel-cache-centos-8 + - publish-wheel-cache-debian-buster + - publish-wheel-cache-ubuntu-focal + - publish-wheel-cache-ubuntu-bionic + - publish-wheel-cache-ubuntu-xenial + - publish-wheel-cache-centos-8-arm64 + - publish-wheel-cache-debian-buster-arm64 + - publish-wheel-cache-ubuntu-focal-arm64 + - publish-wheel-cache-ubuntu-bionic-arm64 + - publish-wheel-cache-ubuntu-xenial-arm64 + - release-wheel-cache: dependencies: - - publish-wheel-mirror-centos-7 - - publish-wheel-mirror-centos-8 - - publish-wheel-mirror-debian-buster - - publish-wheel-mirror-ubuntu-focal - - publish-wheel-mirror-ubuntu-bionic - - publish-wheel-mirror-ubuntu-xenial - - publish-wheel-mirror-centos-8-arm64 - - publish-wheel-mirror-debian-buster-arm64 - - publish-wheel-mirror-ubuntu-focal-arm64 - - publish-wheel-mirror-ubuntu-bionic-arm64 - - publish-wheel-mirror-ubuntu-xenial-arm64 + - publish-wheel-cache-centos-7 + - publish-wheel-cache-centos-8 + - publish-wheel-cache-debian-buster + - publish-wheel-cache-ubuntu-focal + - publish-wheel-cache-ubuntu-bionic + - publish-wheel-cache-ubuntu-xenial + - publish-wheel-cache-centos-8-arm64 + - publish-wheel-cache-debian-buster-arm64 + - publish-wheel-cache-ubuntu-focal-arm64 + - publish-wheel-cache-ubuntu-bionic-arm64 + - publish-wheel-cache-ubuntu-xenial-arm64 - propose-updates: nodeset: ubuntu-bionic branches: master