diff --git a/jenkins/jobs/projects.yaml b/jenkins/jobs/projects.yaml index 79c51b85f4..88e5ee0310 100644 --- a/jenkins/jobs/projects.yaml +++ b/jenkins/jobs/projects.yaml @@ -6893,6 +6893,13 @@ branch-override: stable/liberty - 'gate-{name}-tox-{envlist}': envlist: validate + # This is a periodic job to ensure that our wheels are reasonably + # up-to-date. Adding it to the requirements project seems to be the best + # place to put it, though it's not strictly a requirement for + # requirements. + - wheel-build-{node_arch}: + node_arch: + - ubuntu-trusty-amd64 - project: name: requirements-jobs diff --git a/jenkins/jobs/wheel-mirror.yaml b/jenkins/jobs/wheel-mirror.yaml new file mode 100644 index 0000000000..04ef61a24c --- /dev/null +++ b/jenkins/jobs/wheel-mirror.yaml @@ -0,0 +1,55 @@ +- builder: + name: wheel-build + builders: + - shell: | + #!/bin/bash -xe + + # Generate the AFS Slug from the host system. + source /usr/local/jenkins/slave_scripts/afs-slug.sh + + AFS_DIR=/afs/.openstack.org/mirror/wheel/$AFS_SLUG/ + WHEEL_DIR=/opt/wheel/workspace + + # Delete any previous build directory + rm -rf $WHEEL_DIR + mkdir $WHEEL_DIR + + # Build the wheels into staging directory + echo "Building wheels" + /usr/local/jenkins/slave_scripts/wheel-build.sh $WHEEL_DIR + + # Get an afs token and copy the wheels to AFS + echo "Obtaining token and copying wheels to AFS" + k5start -t -f /etc/wheel.keytab \ + service/wheel \ + -- timeout -k 2m 30m \ + /usr/local/jenkins/slave_scripts/wheel-copy.sh $WHEEL_DIR $AFS_DIR + + # Get an afs token and rebuild the mirror index.html + echo "Obtaining token and rebuilding mirror index." + k5start -t -f /etc/wheel.keytab \ + service/wheel \ + -- timeout -k 2m 30m \ + /usr/local/jenkins/slave_scripts/wheel-index.sh $AFS_DIR + + echo "Done." + +- job-template: + name: 'wheel-build-{node_arch}' + node: 'wheel-mirror-{node_arch}' + + wrappers: + - build-timeout: + timeout: 90 + - timestamps + + builders: + - revoke-sudo + - link-logs + - net-info + - zuul-clone: + project: openstack/requirements + - wheel-build + + publishers: + - console-log diff --git a/jenkins/scripts/afs-slug.sh b/jenkins/scripts/afs-slug.sh new file mode 100644 index 0000000000..4f5b1bed5a --- /dev/null +++ b/jenkins/scripts/afs-slug.sh @@ -0,0 +1,45 @@ +#!/bin/bash -xe + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This script generates a descriptor slug to use with AFS, composed of the +# operating system, its version, and the processor architecture. + +# Pull in the os release. +# ID is 'fedora', 'centos', 'ubuntu' +# VERSION_ID is '23', '7', '14.04' +# Nothing else is useful and/or reliable across distros +source /etc/os-release + +################################################################################ +# Generate an OS Release Name +OS_TYPE=$ID + +################################################################################ +# Generate a version string. +OS_VERSION=$VERSION_ID +if [ "$OS_TYPE" != "ubuntu" ]; then + OS_VERSION=$(echo $OS_VERSION | cut -d'.' -f1) +fi + +################################################################################ +# Get the processor architecture. +# x86_64, i386, armv7l, armv6l +OS_ARCH=$(uname -m) + +################################################################################ +# Build the name +AFS_SLUG="$OS_TYPE-$OS_VERSION-$OS_ARCH" +AFS_SLUG=$(echo "$AFS_SLUG" | tr '[:upper:]' '[:lower:]') + +export AFS_SLUG diff --git a/jenkins/scripts/wheel-build.sh b/jenkins/scripts/wheel-build.sh new file mode 100755 index 0000000000..9f7740f785 --- /dev/null +++ b/jenkins/scripts/wheel-build.sh @@ -0,0 +1,14 @@ +#!/bin/bash -xe + +# Working variables +WHEELHOUSE_DIR=$1 +PROJECT=openstack/requirements +WORKING_DIR=`pwd`/$PROJECT + +# Extract and iterate over all the branch names. +BRANCHES=`git --git-dir=$WORKING_DIR/.git branch -r | grep '^ origin/[^H]'` +for BRANCH in $BRANCHES; do + git --git-dir=$WORKING_DIR/.git show $BRANCH:upper-constraints.txt \ + 2>/dev/null > /tmp/upper-constraints.txt || true + pip wheel -r /tmp/upper-constraints.txt -w $WHEELHOUSE_DIR || true +done diff --git a/jenkins/scripts/wheel-copy.sh b/jenkins/scripts/wheel-copy.sh new file mode 100755 index 0000000000..b24ee6693b --- /dev/null +++ b/jenkins/scripts/wheel-copy.sh @@ -0,0 +1,21 @@ +#!/bin/bash -xe + +# Working variables +WHEELHOUSE_DIR=$1 +MIRROR_ROOT=$2 + +# Build our mirror folder structure. +for f in $WHEELHOUSE_DIR/*; do + BASENAME=$(basename $f) + IFS='-' read -a parts <<< $BASENAME + DEST_DIR="${parts[0]:0:1}/${parts[0]}" + + # Create the mirror directories in AFS /s/split style. This + # depends on the existence of a mod_rewrite script which unmunges + # the path, and is required because AFS has a practical folder size + # limit. + if [ ! -f $MIRROR_ROOT/$DEST_DIR/$BASENAME ]; then + mkdir -p $MIRROR_ROOT/$DEST_DIR + cp $f $MIRROR_ROOT/$DEST_DIR/$BASENAME + fi +done diff --git a/jenkins/scripts/wheel-index.sh b/jenkins/scripts/wheel-index.sh new file mode 100755 index 0000000000..f9afae3c65 --- /dev/null +++ b/jenkins/scripts/wheel-index.sh @@ -0,0 +1,20 @@ +#!/bin/bash -xe + +# Working variables +MIRROR_ROOT=$1 +INDEX_FILE=${MIRROR_ROOT}/index.html + +# Start building our file +echo -e "\n
\n