Merge "Create jobs for a wheel mirror"

jenkins/jobs/projects.yaml

@@ -6868,6 +6868,13 @@
           branch-override: stable/liberty
       - 'gate-{name}-tox-{envlist}':
           envlist: validate
+      # This is a periodic job to ensure that our wheels are reasonably
+      # up-to-date. Adding it to the requirements project seems to be the best
+      # place to put it, though it's not strictly a requirement for
+      # requirements.
+      - wheel-build-{node_arch}:
+          node_arch:
+            - ubuntu-trusty-amd64
 
 - project:
     name: requirements-jobs

jenkins/jobs/wheel-mirror.yaml

@@ -0,0 +1,55 @@
+- builder:
+    name: wheel-build
+    builders:
+      - shell: |
+          #!/bin/bash -xe
+
+          # Generate the AFS Slug from the host system.
+          source /usr/local/jenkins/slave_scripts/afs-slug.sh
+
+          AFS_DIR=/afs/.openstack.org/mirror/wheel/$AFS_SLUG/
+          WHEEL_DIR=/opt/wheel/workspace
+
+          # Delete any previous build directory
+          rm -rf $WHEEL_DIR
+          mkdir $WHEEL_DIR
+
+          # Build the wheels into staging directory
+          echo "Building wheels"
+          /usr/local/jenkins/slave_scripts/wheel-build.sh $WHEEL_DIR
+
+          # Get an afs token and copy the wheels to AFS
+          echo "Obtaining token and copying wheels to AFS"
+          k5start -t -f /etc/wheel.keytab \
+              service/wheel \
+              -- timeout -k 2m 30m \
+              /usr/local/jenkins/slave_scripts/wheel-copy.sh $WHEEL_DIR $AFS_DIR
+
+          # Get an afs token and rebuild the mirror index.html
+          echo "Obtaining token and rebuilding mirror index."
+          k5start -t -f /etc/wheel.keytab \
+              service/wheel \
+              -- timeout -k 2m 30m \
+              /usr/local/jenkins/slave_scripts/wheel-index.sh $AFS_DIR
+
+          echo "Done."
+
+- job-template:
+    name: 'wheel-build-{node_arch}'
+    node: 'wheel-mirror-{node_arch}'
+
+    wrappers:
+      - build-timeout:
+          timeout: 90
+      - timestamps
+
+    builders:
+      - revoke-sudo
+      - link-logs
+      - net-info
+      - zuul-clone:
+          project: openstack/requirements
+      - wheel-build
+
+    publishers:
+      - console-log

jenkins/scripts/afs-slug.sh

@@ -0,0 +1,45 @@
+#!/bin/bash -xe
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This script generates a descriptor slug to use with AFS, composed of the
+# operating system, its version, and the processor architecture.
+
+# Pull in the os release.
+#   ID is 'fedora', 'centos', 'ubuntu'
+#   VERSION_ID is '23', '7', '14.04'
+#   Nothing else is useful and/or reliable across distros
+source /etc/os-release
+
+################################################################################
+# Generate an OS Release Name
+OS_TYPE=$ID
+
+################################################################################
+# Generate a version string.
+OS_VERSION=$VERSION_ID
+if [ "$OS_TYPE" != "ubuntu" ]; then
+    OS_VERSION=$(echo $OS_VERSION | cut -d'.' -f1)
+fi
+
+################################################################################
+# Get the processor architecture.
+#  x86_64, i386, armv7l, armv6l
+OS_ARCH=$(uname -m)
+
+################################################################################
+# Build the name
+AFS_SLUG="$OS_TYPE-$OS_VERSION-$OS_ARCH"
+AFS_SLUG=$(echo "$AFS_SLUG" | tr '[:upper:]' '[:lower:]')
+
+export AFS_SLUG

jenkins/scripts/wheel-build.sh

@@ -0,0 +1,14 @@
+#!/bin/bash -xe
+
+# Working variables
+WHEELHOUSE_DIR=$1
+PROJECT=openstack/requirements
+WORKING_DIR=`pwd`/$PROJECT
+
+# Extract and iterate over all the branch names.
+BRANCHES=`git --git-dir=$WORKING_DIR/.git branch -r | grep '^  origin/[^H]'`
+for BRANCH in $BRANCHES; do
+    git --git-dir=$WORKING_DIR/.git show $BRANCH:upper-constraints.txt \
+        2>/dev/null > /tmp/upper-constraints.txt  || true
+    pip wheel -r /tmp/upper-constraints.txt -w $WHEELHOUSE_DIR || true
+done

jenkins/scripts/wheel-copy.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -xe
+
+# Working variables
+WHEELHOUSE_DIR=$1
+MIRROR_ROOT=$2
+
+# Build our mirror folder structure.
+for f in $WHEELHOUSE_DIR/*; do
+    BASENAME=$(basename $f)
+    IFS='-' read -a parts <<< $BASENAME
+    DEST_DIR="${parts[0]:0:1}/${parts[0]}"
+
+    # Create the mirror directories in AFS /s/split style. This
+    # depends on the existence of a mod_rewrite script which unmunges
+    # the path, and is required because AFS has a practical folder size
+    # limit.
+    if [ ! -f $MIRROR_ROOT/$DEST_DIR/$BASENAME ]; then
+        mkdir -p $MIRROR_ROOT/$DEST_DIR
+        cp $f $MIRROR_ROOT/$DEST_DIR/$BASENAME
+    fi
+done

jenkins/scripts/wheel-index.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -xe
+
+# Working variables
+MIRROR_ROOT=$1
+INDEX_FILE=${MIRROR_ROOT}/index.html
+
+# Start building our file
+echo -e "<html>\n  <head>\n    <title>Wheel Index</title>\n  </head>" > $INDEX_FILE
+echo -e "  <body>\n    <ul>" >> $INDEX_FILE
+
+# Get a list of files
+FILES=`find $MIRROR_ROOT -maxdepth 2 -type d`
+REGEX="([^/])\/($1[^/]+)$"
+for f in $FILES; do
+    if [[ $f =~ $REGEX ]]; then
+        echo "      <li><a href=\"./${BASH_REMATCH[2]}/\">${BASH_REMATCH[2]}</a></li>" >> $INDEX_FILE
+    fi
+done
+
+echo -e "    </ul>\n  </body>\n</html>" >> $INDEX_FILEFILE