From 04ec88d3cb1d22afb30cba553788aa785ac079f8 Mon Sep 17 00:00:00 2001 From: Gregory Haynes Date: Tue, 17 Mar 2015 02:31:32 +0000 Subject: [PATCH] Create jobs for a wheel mirror In order to speed up tests where we install python packages we want to create wheels for packages we depend on. This patch creates four new scripts: * afs-slug, which generates a string version of the platform architecture, derived from the host system's parameters. * wheel-build, which generates all wheels, from all branches, in `pwd`/wheels * wheel-copy, which copies this wheels directory into our AFS file structure. * wheel-index, which rebuilds the index.html file from the directories found in the afs directory. The above scripts are executed in sequence, with afs credentials only loaded where necessary. Additional timeouts have been provided to prevent things from locking. This patch assumes that a mod_rewrite rule will be available on the serving nodes that unmunges the URL. Co-Authored-By: Sean Dague Depends-On: Ie33861dbcb413c34012820fda76f8be94fb1d151 Depends-On: I81c39d420d8ac70def57949ea0d4c323b8797086 Change-Id: I3463fba889239c565c86ffb080adffd7bfdd18f9 --- jenkins/jobs/projects.yaml | 7 +++++ jenkins/jobs/wheel-mirror.yaml | 55 ++++++++++++++++++++++++++++++++++ jenkins/scripts/afs-slug.sh | 45 ++++++++++++++++++++++++++++ jenkins/scripts/wheel-build.sh | 14 +++++++++ jenkins/scripts/wheel-copy.sh | 21 +++++++++++++ jenkins/scripts/wheel-index.sh | 20 +++++++++++++ zuul/layout.yaml | 4 ++- 7 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 jenkins/jobs/wheel-mirror.yaml create mode 100644 jenkins/scripts/afs-slug.sh create mode 100755 jenkins/scripts/wheel-build.sh create mode 100755 jenkins/scripts/wheel-copy.sh create mode 100755 jenkins/scripts/wheel-index.sh diff --git a/jenkins/jobs/projects.yaml b/jenkins/jobs/projects.yaml index 79c51b85f4..88e5ee0310 100644 --- a/jenkins/jobs/projects.yaml +++ b/jenkins/jobs/projects.yaml @@ -6893,6 +6893,13 @@ branch-override: stable/liberty - 'gate-{name}-tox-{envlist}': envlist: validate + # This is a periodic job to ensure that our wheels are reasonably + # up-to-date. Adding it to the requirements project seems to be the best + # place to put it, though it's not strictly a requirement for + # requirements. + - wheel-build-{node_arch}: + node_arch: + - ubuntu-trusty-amd64 - project: name: requirements-jobs diff --git a/jenkins/jobs/wheel-mirror.yaml b/jenkins/jobs/wheel-mirror.yaml new file mode 100644 index 0000000000..04ef61a24c --- /dev/null +++ b/jenkins/jobs/wheel-mirror.yaml @@ -0,0 +1,55 @@ +- builder: + name: wheel-build + builders: + - shell: | + #!/bin/bash -xe + + # Generate the AFS Slug from the host system. + source /usr/local/jenkins/slave_scripts/afs-slug.sh + + AFS_DIR=/afs/.openstack.org/mirror/wheel/$AFS_SLUG/ + WHEEL_DIR=/opt/wheel/workspace + + # Delete any previous build directory + rm -rf $WHEEL_DIR + mkdir $WHEEL_DIR + + # Build the wheels into staging directory + echo "Building wheels" + /usr/local/jenkins/slave_scripts/wheel-build.sh $WHEEL_DIR + + # Get an afs token and copy the wheels to AFS + echo "Obtaining token and copying wheels to AFS" + k5start -t -f /etc/wheel.keytab \ + service/wheel \ + -- timeout -k 2m 30m \ + /usr/local/jenkins/slave_scripts/wheel-copy.sh $WHEEL_DIR $AFS_DIR + + # Get an afs token and rebuild the mirror index.html + echo "Obtaining token and rebuilding mirror index." + k5start -t -f /etc/wheel.keytab \ + service/wheel \ + -- timeout -k 2m 30m \ + /usr/local/jenkins/slave_scripts/wheel-index.sh $AFS_DIR + + echo "Done." + +- job-template: + name: 'wheel-build-{node_arch}' + node: 'wheel-mirror-{node_arch}' + + wrappers: + - build-timeout: + timeout: 90 + - timestamps + + builders: + - revoke-sudo + - link-logs + - net-info + - zuul-clone: + project: openstack/requirements + - wheel-build + + publishers: + - console-log diff --git a/jenkins/scripts/afs-slug.sh b/jenkins/scripts/afs-slug.sh new file mode 100644 index 0000000000..4f5b1bed5a --- /dev/null +++ b/jenkins/scripts/afs-slug.sh @@ -0,0 +1,45 @@ +#!/bin/bash -xe + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This script generates a descriptor slug to use with AFS, composed of the +# operating system, its version, and the processor architecture. + +# Pull in the os release. +# ID is 'fedora', 'centos', 'ubuntu' +# VERSION_ID is '23', '7', '14.04' +# Nothing else is useful and/or reliable across distros +source /etc/os-release + +################################################################################ +# Generate an OS Release Name +OS_TYPE=$ID + +################################################################################ +# Generate a version string. +OS_VERSION=$VERSION_ID +if [ "$OS_TYPE" != "ubuntu" ]; then + OS_VERSION=$(echo $OS_VERSION | cut -d'.' -f1) +fi + +################################################################################ +# Get the processor architecture. +# x86_64, i386, armv7l, armv6l +OS_ARCH=$(uname -m) + +################################################################################ +# Build the name +AFS_SLUG="$OS_TYPE-$OS_VERSION-$OS_ARCH" +AFS_SLUG=$(echo "$AFS_SLUG" | tr '[:upper:]' '[:lower:]') + +export AFS_SLUG diff --git a/jenkins/scripts/wheel-build.sh b/jenkins/scripts/wheel-build.sh new file mode 100755 index 0000000000..9f7740f785 --- /dev/null +++ b/jenkins/scripts/wheel-build.sh @@ -0,0 +1,14 @@ +#!/bin/bash -xe + +# Working variables +WHEELHOUSE_DIR=$1 +PROJECT=openstack/requirements +WORKING_DIR=`pwd`/$PROJECT + +# Extract and iterate over all the branch names. +BRANCHES=`git --git-dir=$WORKING_DIR/.git branch -r | grep '^ origin/[^H]'` +for BRANCH in $BRANCHES; do + git --git-dir=$WORKING_DIR/.git show $BRANCH:upper-constraints.txt \ + 2>/dev/null > /tmp/upper-constraints.txt || true + pip wheel -r /tmp/upper-constraints.txt -w $WHEELHOUSE_DIR || true +done diff --git a/jenkins/scripts/wheel-copy.sh b/jenkins/scripts/wheel-copy.sh new file mode 100755 index 0000000000..b24ee6693b --- /dev/null +++ b/jenkins/scripts/wheel-copy.sh @@ -0,0 +1,21 @@ +#!/bin/bash -xe + +# Working variables +WHEELHOUSE_DIR=$1 +MIRROR_ROOT=$2 + +# Build our mirror folder structure. +for f in $WHEELHOUSE_DIR/*; do + BASENAME=$(basename $f) + IFS='-' read -a parts <<< $BASENAME + DEST_DIR="${parts[0]:0:1}/${parts[0]}" + + # Create the mirror directories in AFS /s/split style. This + # depends on the existence of a mod_rewrite script which unmunges + # the path, and is required because AFS has a practical folder size + # limit. + if [ ! -f $MIRROR_ROOT/$DEST_DIR/$BASENAME ]; then + mkdir -p $MIRROR_ROOT/$DEST_DIR + cp $f $MIRROR_ROOT/$DEST_DIR/$BASENAME + fi +done diff --git a/jenkins/scripts/wheel-index.sh b/jenkins/scripts/wheel-index.sh new file mode 100755 index 0000000000..f9afae3c65 --- /dev/null +++ b/jenkins/scripts/wheel-index.sh @@ -0,0 +1,20 @@ +#!/bin/bash -xe + +# Working variables +MIRROR_ROOT=$1 +INDEX_FILE=${MIRROR_ROOT}/index.html + +# Start building our file +echo -e "\n \n Wheel Index\n " > $INDEX_FILE +echo -e " \n \n \n" >> $INDEX_FILEFILE diff --git a/zuul/layout.yaml b/zuul/layout.yaml index 6cbff1a254..0dfe2ab061 100755 --- a/zuul/layout.yaml +++ b/zuul/layout.yaml @@ -519,7 +519,8 @@ project-templates: - '{name}-pypi-wheel-upload' release: - '{name}-tarball': - - '{name}-pypi-both-upload' + - '{name}-pypi-both-upload': + - wheel-build-ubuntu-trusty-amd64 # Release OpenStack Server packages. - name: openstack-server-release-jobs @@ -9136,6 +9137,7 @@ projects: - gate-grenade-dsvm-partial-ncpu periodic: - propose-requirements-constraints-master + - wheel-build-ubuntu-trusty-amd64 post: - propose-requirements-updates