Create jobs for a wheel mirror

In order to speed up tests where we install python packages we want to
create wheels for packages we depend on.

This patch creates four new scripts:

* afs-slug, which generates a string version of the platform architecture,
  derived from the host system's parameters.
* wheel-build, which generates all wheels, from all branches, in `pwd`/wheels
* wheel-copy, which copies this wheels directory into our AFS file structure.
* wheel-index, which rebuilds the index.html file from the directories
  found in the afs directory.

The above scripts are executed in sequence, with afs credentials only loaded
where necessary. Additional timeouts have been provided to prevent things from
locking.

This patch assumes that a mod_rewrite rule will be available on the serving nodes
that unmunges the URL.

Co-Authored-By: Sean Dague <sean@dague.net>
Depends-On: Ie33861dbcb413c34012820fda76f8be94fb1d151
Depends-On: I81c39d420d8ac70def57949ea0d4c323b8797086
Change-Id: I3463fba889239c565c86ffb080adffd7bfdd18f9

jenkins/jobs/projects.yaml

@@ -6893,6 +6893,13 @@
           branch-override: stable/liberty
       - 'gate-{name}-tox-{envlist}':
           envlist: validate
+      # This is a periodic job to ensure that our wheels are reasonably
+      # up-to-date. Adding it to the requirements project seems to be the best
+      # place to put it, though it's not strictly a requirement for
+      # requirements.
+      - wheel-build-{node_arch}:
+          node_arch:
+            - ubuntu-trusty-amd64
 
 - project:
     name: requirements-jobs

jenkins/jobs/wheel-mirror.yaml

@@ -0,0 +1,55 @@
+- builder:
+    name: wheel-build
+    builders:
+      - shell: |
+          #!/bin/bash -xe
+
+          # Generate the AFS Slug from the host system.
+          source /usr/local/jenkins/slave_scripts/afs-slug.sh
+
+          AFS_DIR=/afs/.openstack.org/mirror/wheel/$AFS_SLUG/
+          WHEEL_DIR=/opt/wheel/workspace
+
+          # Delete any previous build directory
+          rm -rf $WHEEL_DIR
+          mkdir $WHEEL_DIR
+
+          # Build the wheels into staging directory
+          echo "Building wheels"
+          /usr/local/jenkins/slave_scripts/wheel-build.sh $WHEEL_DIR
+
+          # Get an afs token and copy the wheels to AFS
+          echo "Obtaining token and copying wheels to AFS"
+          k5start -t -f /etc/wheel.keytab \
+              service/wheel \
+              -- timeout -k 2m 30m \
+              /usr/local/jenkins/slave_scripts/wheel-copy.sh $WHEEL_DIR $AFS_DIR
+
+          # Get an afs token and rebuild the mirror index.html
+          echo "Obtaining token and rebuilding mirror index."
+          k5start -t -f /etc/wheel.keytab \
+              service/wheel \
+              -- timeout -k 2m 30m \
+              /usr/local/jenkins/slave_scripts/wheel-index.sh $AFS_DIR
+
+          echo "Done."
+
+- job-template:
+    name: 'wheel-build-{node_arch}'
+    node: 'wheel-mirror-{node_arch}'
+
+    wrappers:
+      - build-timeout:
+          timeout: 90
+      - timestamps
+
+    builders:
+      - revoke-sudo
+      - link-logs
+      - net-info
+      - zuul-clone:
+          project: openstack/requirements
+      - wheel-build
+
+    publishers:
+      - console-log

jenkins/scripts/afs-slug.sh

@@ -0,0 +1,45 @@
+#!/bin/bash -xe
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This script generates a descriptor slug to use with AFS, composed of the
+# operating system, its version, and the processor architecture.
+
+# Pull in the os release.
+#   ID is 'fedora', 'centos', 'ubuntu'
+#   VERSION_ID is '23', '7', '14.04'
+#   Nothing else is useful and/or reliable across distros
+source /etc/os-release
+
+################################################################################
+# Generate an OS Release Name
+OS_TYPE=$ID
+
+################################################################################
+# Generate a version string.
+OS_VERSION=$VERSION_ID
+if [ "$OS_TYPE" != "ubuntu" ]; then
+    OS_VERSION=$(echo $OS_VERSION | cut -d'.' -f1)
+fi
+
+################################################################################
+# Get the processor architecture.
+#  x86_64, i386, armv7l, armv6l
+OS_ARCH=$(uname -m)
+
+################################################################################
+# Build the name
+AFS_SLUG="$OS_TYPE-$OS_VERSION-$OS_ARCH"
+AFS_SLUG=$(echo "$AFS_SLUG" | tr '[:upper:]' '[:lower:]')
+
+export AFS_SLUG

jenkins/scripts/wheel-build.sh

@@ -0,0 +1,14 @@
+#!/bin/bash -xe
+
+# Working variables
+WHEELHOUSE_DIR=$1
+PROJECT=openstack/requirements
+WORKING_DIR=`pwd`/$PROJECT
+
+# Extract and iterate over all the branch names.
+BRANCHES=`git --git-dir=$WORKING_DIR/.git branch -r | grep '^  origin/[^H]'`
+for BRANCH in $BRANCHES; do
+    git --git-dir=$WORKING_DIR/.git show $BRANCH:upper-constraints.txt \
+        2>/dev/null > /tmp/upper-constraints.txt  || true
+    pip wheel -r /tmp/upper-constraints.txt -w $WHEELHOUSE_DIR || true
+done

jenkins/scripts/wheel-copy.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -xe
+
+# Working variables
+WHEELHOUSE_DIR=$1
+MIRROR_ROOT=$2
+
+# Build our mirror folder structure.
+for f in $WHEELHOUSE_DIR/*; do
+    BASENAME=$(basename $f)
+    IFS='-' read -a parts <<< $BASENAME
+    DEST_DIR="${parts[0]:0:1}/${parts[0]}"
+
+    # Create the mirror directories in AFS /s/split style. This
+    # depends on the existence of a mod_rewrite script which unmunges
+    # the path, and is required because AFS has a practical folder size
+    # limit.
+    if [ ! -f $MIRROR_ROOT/$DEST_DIR/$BASENAME ]; then
+        mkdir -p $MIRROR_ROOT/$DEST_DIR
+        cp $f $MIRROR_ROOT/$DEST_DIR/$BASENAME
+    fi
+done

jenkins/scripts/wheel-index.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -xe
+
+# Working variables
+MIRROR_ROOT=$1
+INDEX_FILE=${MIRROR_ROOT}/index.html
+
+# Start building our file
+echo -e "<html>\n  <head>\n    <title>Wheel Index</title>\n  </head>" > $INDEX_FILE
+echo -e "  <body>\n    <ul>" >> $INDEX_FILE
+
+# Get a list of files
+FILES=`find $MIRROR_ROOT -maxdepth 2 -type d`
+REGEX="([^/])\/($1[^/]+)$"
+for f in $FILES; do
+    if [[ $f =~ $REGEX ]]; then
+        echo "      <li><a href=\"./${BASH_REMATCH[2]}/\">${BASH_REMATCH[2]}</a></li>" >> $INDEX_FILE
+    fi
+done
+
+echo -e "    </ul>\n  </body>\n</html>" >> $INDEX_FILEFILE

zuul/layout.yaml

@@ -519,7 +519,8 @@ project-templates:
           - '{name}-pypi-wheel-upload'
     release:
       - '{name}-tarball':
-        - '{name}-pypi-both-upload'
+        - '{name}-pypi-both-upload':
+            - wheel-build-ubuntu-trusty-amd64
 
   # Release OpenStack Server packages.
   - name: openstack-server-release-jobs
@@ -9136,6 +9137,7 @@ projects:
       - gate-grenade-dsvm-partial-ncpu
     periodic:
       - propose-requirements-constraints-master
+      - wheel-build-ubuntu-trusty-amd64
     post:
       - propose-requirements-updates