Merge "Move wheel build role/job outside project-config"

This commit is contained in:
Zuul 2020-06-03 22:46:09 +00:00 committed by Gerrit Code Review
commit 7329ceaa67
8 changed files with 48 additions and 380 deletions

View File

@ -1,10 +0,0 @@
- hosts: all
strategy: free
pre_tasks:
- name: Set fact for wheel dir
set_fact:
wheel_dir: /opt/wheel/workspace
roles:
- build-wheels

View File

@ -1,14 +0,0 @@
- hosts: all
tasks:
- name: Ensure build logs directory exists
file:
path: "{{ zuul.executor.log_root }}/{{ wheel_python }}"
state: directory
delegate_to: localhost
- name: Collect build logs
synchronize:
dest: "{{ zuul.executor.log_root }}/{{ wheel_python }}"
mode: pull
src: ~/logs/
verify_host: true

View File

@ -1,26 +0,0 @@
- hosts: all
roles:
- bindep
pre_tasks:
# NOTE(pabelanger): Manually install EPEL dependencies as not to
# keep EPEL enabled.
- name: Manually install EPEL dependencies
become: yes
yum:
name: "{{ item }}"
enablerepo: epel
state: present
loop:
- parallel
when: ansible_distribution == "CentOS"
# NOTE(pabelanger): We need to install extra libraries, such as
# liberasurecode-devel
# which are only part of the RDO repositories.
- name: Add centos-release-openstack-queens support
yum:
name: centos-release-openstack-queens
state: present
become: yes
when: ansible_distribution == "CentOS"

View File

@ -1 +0,0 @@
Run the build scripts on the wheel mirror builders

View File

@ -1,90 +0,0 @@
#!/bin/bash -xe
# Working variables
WHEELHOUSE_DIR=$1
WORKING_DIR=$(pwd)/src/opendev.org/openstack/requirements
PYTHON_VERSION=$2
LOGS=$(pwd)/logs
FAIL_LOG=${LOGS}/failed.txt
# preclean logs
mkdir -p ${LOGS}
rm -rf ${LOGS}/*
# Extract and iterate over all the branch names.
BRANCHES=`git --git-dir=$WORKING_DIR/.git branch -a | grep '^ stable' | \
grep -Ev '(newton)'`
for BRANCH in master $BRANCHES; do
git --git-dir=$WORKING_DIR/.git show $BRANCH:upper-constraints.txt \
2>/dev/null > /tmp/upper-constraints.txt || true
# setup the building virtualenv. We want to freshen this for each
# branch.
rm -rf build_env
virtualenv -p $PYTHON_VERSION build_env
build_env/bin/pip install --upgrade pip
# SHORT_BRANCH is just "master","newton","kilo" etc. because this
# keeps the output log hierarchy much simpler.
SHORT_BRANCH=${BRANCH##origin/}
SHORT_BRANCH=${SHORT_BRANCH##stable/}
# Failed parallel jobs don't fail the whole job, we just report
# the issues for investigation.
set +e
# This runs all the jobs under "parallel". The stdout, stderr and
# exit status for each pip invocation will be captured into files
# kept in ${LOGS}/build/${SHORT_BRANCH}/1/[package]. The --joblog
# file keeps an overview of all run jobs, which we can probe to
# find failed jobs.
cat /tmp/upper-constraints.txt | \
parallel --files --progress --joblog ${LOGS}/$SHORT_BRANCH-job.log \
--results ${LOGS}/build/$SHORT_BRANCH \
build_env/bin/pip --verbose --exists-action=i wheel \
-c /tmp/upper-constraints.txt \
-w $WHEELHOUSE_DIR {}
set -e
# Column $7 is the exit status of the job, $NF is the last
# argument to pip, which is our package.
FAILED=$(awk -e '$7!=0 {print $NF}' ${LOGS}/$SHORT_BRANCH-job.log)
if [ -n "${FAILED}" ]; then
echo "*** FAILED BUILDS FOR BRANCH ${BRANCH}" >> ${FAIL_LOG}
echo "${FAILED}" >> ${FAIL_LOG}
echo -e "***\n\n" >> ${FAIL_LOG}
fi
done
# Parse stdout from all the build logs to build up a unique list of all
# wheels downloaded from PyPI and delete them from the wheelhouse, since
# this is only meant to provide built wheels which are absent from PyPI.
find ${LOGS}/build/ -name stdout -exec grep 'Downloading' {} \; \
| sed -n 's,.*Downloading.*[ /]\([^ /]*\.whl\)\([ #].*\|$\),\1,p' \
| sort -u > ${LOGS}/remove-wheels.txt
pushd ${WHEELHOUSE_DIR}
cat ${LOGS}/remove-wheels.txt | xargs rm
popd
if [ -f ${FAIL_LOG} ]; then
cat ${FAIL_LOG}
fi
# XXX This does make a lot of log files; about 80mb after compression.
# In theory we could correlate just the failed logs and keep those
# from the failure logs above. This is currently (2017-01) left as an
# exercise for when the job is stable :) bz2 gave about 20%
# improvement over gzip in testing.
pushd ${LOGS}
tar zcvf build-logs.tar.gz ./build
rm -rf ./build
popd
# Set the final exit status to 1 if remove-wheels.txt is empty so the
# job will fail.
if [ ! -s ${LOGS}/remove-wheels.txt ]; then
echo "*** EMPTY WHEEL REMOVAL LIST: THIS SHOULD NOT HAPPEN ***"
exit 1
fi

View File

@ -1,17 +0,0 @@
- name: Ensure wheel dir exists
file:
group: zuul
owner: zuul
path: "{{ wheel_dir }}"
state: directory
become: yes
- name: Prevent using existing wheel mirror
become: yes
lineinfile:
path: /etc/pip.conf
state: absent
regexp: '^extra-index-url.*$'
- name: Build the wheel mirror
script: wheel-build.sh {{ wheel_dir }} {{ wheel_python }}

View File

@ -776,7 +776,8 @@
- job:
name: openstack-upload-github-mirror
parent: opendev-upload-git-mirror
description: Mirrors official projects to github's openstack org
description: |
Mirrors official projects to github's openstack org
final: true
protected: true
secrets:
@ -817,31 +818,10 @@
protected: true
- job:
# NOTE(ianw) : We are in the process of removing the
# "wheel-mirror" jobs in project-config as we have moved them into
# openstack-zuul-jobs. This is just until requirements removes
# all references to the old jobs.
name: build-wheel-mirror-base
parent: build-wheel-cache-base
- job:
name: build-wheel-mirror-centos-7
parent: build-wheel-mirror-base
description: |
Build CentOS 7 wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-centos-7-python2
label: centos-7
host-vars:
wheel-mirror-centos-7-python2:
wheel_python: python2
- job:
name: publish-wheel-mirror-centos-7
name: publish-wheel-cache-centos-7
description: |
Publish CentOS 7 wheels for OpenStack CI mirrors.
parent: build-wheel-mirror-centos-7
parent: build-wheel-cache-centos-7
pre-run: playbooks/openafs-client/setup.yaml
post-run: playbooks/publish/wheel-mirror.yaml
allowed-projects:
@ -852,23 +832,10 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-centos-8
parent: build-wheel-mirror-base
description: |
Build CentOS 8 wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-centos-8-python3
label: centos-8
host-vars:
wheel-mirror-centos-8-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-centos-8
name: publish-wheel-cache-centos-8
description: |
Publish CentOS 8 wheels for OpenStack CI mirrors.
parent: build-wheel-mirror-centos-8
parent: build-wheel-cache-centos-8
pre-run: playbooks/openafs-client/setup.yaml
post-run: playbooks/publish/wheel-mirror.yaml
allowed-projects:
@ -879,23 +846,10 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-centos-8-arm64
parent: build-wheel-mirror-base
description: |
Build CentOS 8 wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-centos-8-arm64-python3
label: centos-8-arm64
host-vars:
wheel-mirror-centos-8-arm64-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-centos-8-arm64
name: publish-wheel-cache-centos-8-arm64
description: |
Publish CentOS 8 wheels for OpenStack CI mirrors.
parent: build-wheel-mirror-centos-8-arm64
parent: build-wheel-cache-centos-8-arm64
pre-run: playbooks/openafs-client/setup.yaml
post-run: playbooks/publish/wheel-mirror.yaml
allowed-projects:
@ -906,27 +860,10 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-debian-buster
parent: build-wheel-mirror-base
description: |
Build Debian buster wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-debian-buster-python2
label: debian-buster
- name: wheel-mirror-debian-buster-python3
label: debian-buster
host-vars:
wheel-mirror-debian-buster-python2:
wheel_python: python2
wheel-mirror-debian-buster-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-debian-buster
name: publish-wheel-cache-debian-buster
description: |
Publish Debian buster wheels for OpenStack CI mirrors.
parent: build-wheel-mirror-debian-buster
parent: build-wheel-cache-debian-buster
pre-run: playbooks/openafs-client/setup.yaml
post-run: playbooks/publish/wheel-mirror.yaml
allowed-projects:
@ -937,27 +874,10 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-debian-buster-arm64
parent: build-wheel-mirror-base
description: |
Build Debian buster wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-debian-buster-arm64-python2
label: debian-buster-arm64
- name: wheel-mirror-debian-buster-arm64-python3
label: debian-buster-arm64
host-vars:
wheel-mirror-debian-buster-arm64-python2:
wheel_python: python2
wheel-mirror-debian-buster-arm64-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-debian-buster-arm64
name: publish-wheel-cache-debian-buster-arm64
description: |
Publish Debian buster wheels for OpenStack CI mirrors.
parent: build-wheel-mirror-debian-buster-arm64
parent: build-wheel-cache-debian-buster-arm64
pre-run: playbooks/openafs-client/setup.yaml
post-run: playbooks/publish/wheel-mirror.yaml
allowed-projects:
@ -968,21 +888,8 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-ubuntu-focal
parent: build-wheel-mirror-base
description: |
Build Ubuntu Focal wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-ubuntu-focal-python3
label: ubuntu-focal
host-vars:
wheel-mirror-ubuntu-focal-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-ubuntu-focal
parent: build-wheel-mirror-ubuntu-focal
name: publish-wheel-cache-ubuntu-focal
parent: build-wheel-cache-ubuntu-focal
description: |
Publish Ubuntu Focal wheels for OpenStack CI mirrors.
pre-run: playbooks/openafs-client/setup.yaml
@ -995,21 +902,8 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-ubuntu-focal-arm64
parent: build-wheel-mirror-base
description: |
Build Ubuntu Focal wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-ubuntu-focal-arm64-python3
label: ubuntu-focal-arm64
host-vars:
wheel-mirror-ubuntu-focal-arm64-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-ubuntu-focal-arm64
parent: build-wheel-mirror-ubuntu-focal-arm64
name: publish-wheel-cache-ubuntu-focal-arm64
parent: build-wheel-cache-ubuntu-focal-arm64
description: |
Publish Ubuntu Focal wheels for OpenStack CI mirrors.
pre-run: playbooks/openafs-client/setup.yaml
@ -1022,25 +916,8 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-ubuntu-bionic
parent: build-wheel-mirror-base
description: |
Build Ubuntu Bionic wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-ubuntu-bionic-python2
label: ubuntu-bionic
- name: wheel-mirror-ubuntu-bionic-python3
label: ubuntu-bionic
host-vars:
wheel-mirror-ubuntu-bionic-python2:
wheel_python: python2
wheel-mirror-ubuntu-bionic-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-ubuntu-bionic
parent: build-wheel-mirror-ubuntu-bionic
name: publish-wheel-cache-ubuntu-bionic
parent: build-wheel-cache-ubuntu-bionic
description: |
Publish Ubuntu Bionic wheels for OpenStack CI mirrors.
pre-run: playbooks/openafs-client/setup.yaml
@ -1053,25 +930,8 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-ubuntu-bionic-arm64
parent: build-wheel-mirror-base
description: |
Build Ubuntu Bionic wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-ubuntu-bionic-arm64-python2
label: ubuntu-bionic-arm64
- name: wheel-mirror-ubuntu-bionic-arm64-python3
label: ubuntu-bionic-arm64
host-vars:
wheel-mirror-ubuntu-bionic-arm64-python2:
wheel_python: python2
wheel-mirror-ubuntu-bionic-arm64-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-ubuntu-bionic-arm64
parent: build-wheel-mirror-ubuntu-bionic-arm64
name: publish-wheel-cache-ubuntu-bionic-arm64
parent: build-wheel-cache-ubuntu-bionic-arm64
description: |
Publish Ubuntu Bionic wheels for OpenStack CI mirrors.
pre-run: playbooks/openafs-client/setup.yaml
@ -1084,25 +944,8 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-ubuntu-xenial
parent: build-wheel-mirror-base
description: |
Build Ubuntu Xenial wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-ubuntu-xenial-python2
label: ubuntu-xenial
- name: wheel-mirror-ubuntu-xenial-python3
label: ubuntu-xenial
host-vars:
wheel-mirror-ubuntu-xenial-python2:
wheel_python: python2
wheel-mirror-ubuntu-xenial-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-ubuntu-xenial
parent: build-wheel-mirror-ubuntu-xenial
name: publish-wheel-cache-ubuntu-xenial
parent: build-wheel-cache-ubuntu-xenial
description: |
Publish Ubuntu Xenial wheels for OpenStack CI mirrors.
pre-run: playbooks/openafs-client/setup.yaml
@ -1115,25 +958,8 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-ubuntu-xenial-arm64
parent: build-wheel-mirror-base
description: |
Build Ubuntu Xenial ARM64 wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-ubuntu-xenial-arm64-python2
label: ubuntu-xenial-arm64
- name: wheel-mirror-ubuntu-xenial-arm64-python3
label: ubuntu-xenial-arm64
host-vars:
wheel-mirror-ubuntu-xenial-arm64-python2:
wheel_python: python2
wheel-mirror-ubuntu-xenial-arm64-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-ubuntu-xenial-arm64
parent: build-wheel-mirror-ubuntu-xenial-arm64
name: publish-wheel-cache-ubuntu-xenial-arm64
parent: build-wheel-cache-ubuntu-xenial-arm64
description: |
Publish Ubuntu Xenial ARM64 wheels for OpenStack CI mirrors.
pre-run: playbooks/openafs-client/setup.yaml
@ -1146,7 +972,7 @@
secret: wheel_keytab
- job:
name: release-wheel-mirror
name: release-wheel-cache
description: |
Release published wheels to OpenStack CI mirrors.
run: playbooks/wheel/release.yaml

View File

@ -5411,30 +5411,30 @@
branches: master
periodic:
jobs:
- publish-wheel-mirror-centos-7
- publish-wheel-mirror-centos-8
- publish-wheel-mirror-debian-buster
- publish-wheel-mirror-ubuntu-focal
- publish-wheel-mirror-ubuntu-bionic
- publish-wheel-mirror-ubuntu-xenial
- publish-wheel-mirror-centos-8-arm64
- publish-wheel-mirror-debian-buster-arm64
- publish-wheel-mirror-ubuntu-focal-arm64
- publish-wheel-mirror-ubuntu-bionic-arm64
- publish-wheel-mirror-ubuntu-xenial-arm64
- release-wheel-mirror:
- publish-wheel-cache-centos-7
- publish-wheel-cache-centos-8
- publish-wheel-cache-debian-buster
- publish-wheel-cache-ubuntu-focal
- publish-wheel-cache-ubuntu-bionic
- publish-wheel-cache-ubuntu-xenial
- publish-wheel-cache-centos-8-arm64
- publish-wheel-cache-debian-buster-arm64
- publish-wheel-cache-ubuntu-focal-arm64
- publish-wheel-cache-ubuntu-bionic-arm64
- publish-wheel-cache-ubuntu-xenial-arm64
- release-wheel-cache:
dependencies:
- publish-wheel-mirror-centos-7
- publish-wheel-mirror-centos-8
- publish-wheel-mirror-debian-buster
- publish-wheel-mirror-ubuntu-focal
- publish-wheel-mirror-ubuntu-bionic
- publish-wheel-mirror-ubuntu-xenial
- publish-wheel-mirror-centos-8-arm64
- publish-wheel-mirror-debian-buster-arm64
- publish-wheel-mirror-ubuntu-focal-arm64
- publish-wheel-mirror-ubuntu-bionic-arm64
- publish-wheel-mirror-ubuntu-xenial-arm64
- publish-wheel-cache-centos-7
- publish-wheel-cache-centos-8
- publish-wheel-cache-debian-buster
- publish-wheel-cache-ubuntu-focal
- publish-wheel-cache-ubuntu-bionic
- publish-wheel-cache-ubuntu-xenial
- publish-wheel-cache-centos-8-arm64
- publish-wheel-cache-debian-buster-arm64
- publish-wheel-cache-ubuntu-focal-arm64
- publish-wheel-cache-ubuntu-bionic-arm64
- publish-wheel-cache-ubuntu-xenial-arm64
- propose-updates:
nodeset: ubuntu-bionic
branches: master