From 0ed04ca11cc2d3cfcdffdcf92eb7946e3db4eb25 Mon Sep 17 00:00:00 2001 From: Don Penney Date: Mon, 22 Oct 2018 16:13:17 -0400 Subject: [PATCH] Add tools to build python wheels and wheel tarball This update adds tools for building wheels for upstream python modules, and generating a StarlingX wheels tarball. Building upstream python modules is done within a docker container, with a wheels.cfg configuration file that identifies the upstream sources and expected python wheel output. The build-base-wheels.sh tool will create the wheel builder image, launch it, and run the build script. The get-stx-wheels.sh tool will extract the python wheels built by the StarlingX load. The build-wheel-tarball.sh script will combine the wheels from the base and StarlingX wheel sets into a single tarball that can be used when building container images for StarlingX services. Story: 2003907 Task: 27531 Change-Id: Icf6c63699a635d1173dcef9eb1d12ae7c4e39108 Signed-off-by: Don Penney --- build-tools/build-wheels/build-base-wheels.sh | 146 +++++++++ .../build-wheels/build-wheel-tarball.sh | 281 ++++++++++++++++ .../build-wheels/docker/centos-dockerfile | 18 ++ .../build-wheels/docker/docker-build-wheel.sh | 301 ++++++++++++++++++ .../build-wheels/docker/pike-wheels.cfg | 229 +++++++++++++ build-tools/build-wheels/get-stx-wheels.sh | 135 ++++++++ 6 files changed, 1110 insertions(+) create mode 100755 build-tools/build-wheels/build-base-wheels.sh create mode 100755 build-tools/build-wheels/build-wheel-tarball.sh create mode 100644 build-tools/build-wheels/docker/centos-dockerfile create mode 100755 build-tools/build-wheels/docker/docker-build-wheel.sh create mode 100644 build-tools/build-wheels/docker/pike-wheels.cfg create mode 100755 build-tools/build-wheels/get-stx-wheels.sh diff --git a/build-tools/build-wheels/build-base-wheels.sh b/build-tools/build-wheels/build-base-wheels.sh new file mode 100755 index 00000000..79e79710 --- /dev/null +++ b/build-tools/build-wheels/build-base-wheels.sh @@ -0,0 +1,146 @@ +#!/bin/bash +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# This utility sets up a docker image to build wheels +# for a set of upstream python modules. +# + +# Required env vars +if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then + echo "Environment not setup for builds" >&2 + exit 1 +fi + +DOCKER_PATH=${MY_REPO}/build-tools/build-wheels/docker +WHEELS_CFG=${DOCKER_PATH}/wheels.cfg +KEEP_IMAGE=no +KEEP_CONTAINER=no +OS=centos +OS_RELEASE=pike + +function usage { + cat >&2 < ] [ --keep-image ] [ --keep-container ] [ --release ] + +Options: + --os: Specify base OS (eg. centos) + --keep-image: Skip deletion of the wheel build image in docker + --keep-container: Skip deletion of container used for the build + --release: Openstack release (default: pike) + +EOF +} + +OPTS=$(getopt -o h -l help,os:,keep-image,keep-container,release: -- "$@") +if [ $? -ne 0 ]; then + usage + exit 1 +fi + +eval set -- "${OPTS}" + +while true; do + case $1 in + --) + # End of getopt arguments + shift + break + ;; + --os) + OS=$2 + shift 2 + ;; + --keep-image) + KEEP_IMAGE=yes + shift + ;; + --keep-container) + KEEP_CONTAINER=yes + shift + ;; + --release) + OS_RELEASE=$2 + shift 2 + ;; + -h | --help ) + usage + exit 1 + ;; + *) + usage + exit 1 + ;; + esac +done + +BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${OS_RELEASE}/base +BUILD_IMAGE_NAME="${USER}-$(basename ${MY_WORKSPACE})-wheelbuilder:${OS}-${OS_RELEASE}" + +DOCKER_FILE=${DOCKER_PATH}/${OS}-dockerfile + +function supported_os_list { + for f in ${DOCKER_PATH}/*-dockerfile; do + echo $(basename ${f%-dockerfile}) + done | xargs echo +} + +if [ ! -f ${DOCKER_FILE} ]; then + echo "Unsupported OS specified: ${OS}" >&2 + echo "Supported OS options: $(supported_os_list)" >&2 + exit 1 +fi + +# +# Check build output directory for unexpected files, +# ie. wheels from old builds that are no longer in wheels.cfg +# +if [ -d ${BUILD_OUTPUT_PATH} ]; then + + for f in ${BUILD_OUTPUT_PATH}/*; do + grep -q "^$(basename $f)|" ${WHEELS_CFG} + if [ $? -ne 0 ]; then + echo "Deleting stale file: $f" + rm -f $f + fi + done +else + mkdir -p ${BUILD_OUTPUT_PATH} + if [ $? -ne 0 ]; then + echo "Failed to create directory: ${BUILD_OUTPUT_PATH}" >&2 + exit 1 + fi +fi + +# Create the builder image +docker build --build-arg OS_RELEASE=${OS_RELEASE} -t ${BUILD_IMAGE_NAME} -f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH} +if [ $? -ne 0 ]; then + echo "Failed to create build image in docker" >&2 + exit 1 +fi + +# Run the image, executing the build-wheel.sh script +RM_OPT= +if [ "${KEEP_CONTAINER}" = "no" ]; then + RM_OPT="--rm" +fi +docker run ${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels -i -t ${BUILD_IMAGE_NAME} /docker-build-wheel.sh + +if [ "${KEEP_IMAGE}" = "no" ]; then + # Delete the builder image + echo "Removing docker build image ${BUILD_IMAGE_NAME}" + docker image rm ${BUILD_IMAGE_NAME} + if [ $? -ne 0 ]; then + echo "Failed to delete build image from docker" >&2 + fi +fi + +# Check for failures +if [ -f ${BUILD_OUTPUT_PATH}/failed.lst ]; then + # Failures would already have been reported + exit 1 +fi + diff --git a/build-tools/build-wheels/build-wheel-tarball.sh b/build-tools/build-wheels/build-wheel-tarball.sh new file mode 100755 index 00000000..b2d184a6 --- /dev/null +++ b/build-tools/build-wheels/build-wheel-tarball.sh @@ -0,0 +1,281 @@ +#!/bin/bash +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# This utility builds the StarlingX wheel tarball +# + +MY_SCRIPT_DIR=$(dirname $(readlink -f $0)) + +# Required env vars +if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then + echo "Environment not setup for builds" >&2 + exit 1 +fi + +SUPPORTED_OS_ARGS=('centos') + +function usage { + cat >&2 < ] [ --push ] [ --user ] [ --release ] + +Options: + --os: Specify base OS (valid options: ${SUPPORTED_OS_ARGS[@]}) + --push: Push to docker repo + --user: Docker repo userid + --release: Openstack release (default: pike) + +EOF +} + +# +# Function to get an auth token from docker +# +function get_docker_token { + local auth_server="auth.docker.io" + local registry="registry.docker.io" + local scope="repository:${REPO_USER}/${IMAGE_NAME}:pull" + curl -k -sSL -X GET "https://${auth_server}/token?service=${registry}&scope=${scope}" \ + | python -c ' +import sys, yaml, json +y=yaml.load(sys.stdin.read()) +print y["token"] +' 2>/dev/null +} + +# +# Function to query docker for the highest current version +# +function get_current_version { + local registry_server="registry-1.docker.io" + local token= + token=$(get_docker_token) + + curl -H "Authorization: Bearer ${token}" -k -sSL -X GET https://${registry_server}/v2/${REPO_USER}/${IMAGE_NAME}/tags/list \ + | python -c ' +import sys, yaml, json, re +y=yaml.load(sys.stdin.read()) +print max([float(v) for v in filter(lambda x: re.search(r"\d+\.\d+", x), y["tags"])]) +' 2>/dev/null +} + +# +# Function to increment the image version above the current docker version +# +function increment_version { + local CUR_VERSION= + CUR_VERSION=$(get_current_version) + + if [ -z "${CUR_VERSION}" ]; then + echo "0.1" + return + fi + + echo "${CUR_VERSION}" | while IFS='.' read x y; do + let -i y++ + echo "${x}.${y}" + done +} + +OPTS=$(getopt -o h -l help,os:,push,user:,release: -- "$@") +if [ $? -ne 0 ]; then + usage + exit 1 +fi + +OS=centos +OS_RELEASE=pike +PUSH=no +REPO_USER=${USER} +# List of top-level services for images, which should not be listed in upper-constraints.txt +SKIP_CONSTRAINTS=(ceilometer cinder glance gnocchi heat horizon keystone neutron nova) + +eval set -- "${OPTS}" + +while true; do + case $1 in + --) + # End of getopt arguments + shift + break + ;; + --os) + OS=$2 + shift 2 + ;; + --push) + PUSH=yes + shift + ;; + --user) + REPO_USER=$2 + shift 2 + ;; + --release) + OS_RELEASE=$2 + shift 2 + ;; + -h | --help ) + usage + exit 1 + ;; + *) + usage + exit 1 + ;; + esac +done + +# Validate the OS option +VALID_OS=1 +for supported_os in ${SUPPORTED_OS_ARGS[@]}; do + if [ "$OS" = "${supported_os}" ]; then + VALID_OS=0 + break + fi +done +if [ ${VALID_OS} -ne 0 ]; then + echo "Unsupported OS specified: ${OS}" >&2 + echo "Supported OS options: ${SUPPORTED_OS_ARGS[@]}" >&2 + exit 1 +fi + +# Build the base wheels and retrieve the StarlingX wheels +${MY_SCRIPT_DIR}/build-base-wheels.sh --os ${OS} --release ${OS_RELEASE} +if [ $? -ne 0 ]; then + echo "Failure running build-base-wheels.sh" >&2 + exit 1 +fi + +${MY_SCRIPT_DIR}/get-stx-wheels.sh --os ${OS} --release ${OS_RELEASE} +if [ $? -ne 0 ]; then + echo "Failure running get-stx-wheels.sh" >&2 + exit 1 +fi + +BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${OS_RELEASE}/tarball +if [ -d ${BUILD_OUTPUT_PATH} ]; then + # Wipe out the existing dir to ensure there are no stale files + rm -rf ${BUILD_OUTPUT_PATH} +fi +mkdir -p ${BUILD_OUTPUT_PATH} +cd ${BUILD_OUTPUT_PATH} + +IMAGE_NAME=stx-${OS}-${OS_RELEASE}-wheels + +TARBALL_FNAME=${MY_WORKSPACE}/std/build-wheels-${OS}-${OS_RELEASE}/${IMAGE_NAME}.tar +if [ -f ${TARBALL_FNAME} ]; then + rm -f ${TARBALL_FNAME} +fi + +# Download the global-requirements.txt and upper-constraints.txt files +wget https://raw.githubusercontent.com/openstack/requirements/stable/${OS_RELEASE}/global-requirements.txt +if [ $? -ne 0 ]; then + echo "Failed to download global-requirements.txt" >&2 + exit 1 +fi + +wget https://raw.githubusercontent.com/openstack/requirements/stable/${OS_RELEASE}/upper-constraints.txt +if [ $? -ne 0 ]; then + echo "Failed to download upper-constraints.txt" >&2 + exit 1 +fi + +# Copy the base and stx wheels, updating upper-constraints.txt as necessary +for wheel in ../base/*.whl ../stx/wheels/*.whl; do + # Get the wheel name and version from the METADATA + METADATA=$(unzip -p ${wheel} '*/METADATA') + name=$(echo "${METADATA}" | grep '^Name:' | awk '{print $2}') + version=$(echo "${METADATA}" | grep '^Version:' | awk '{print $2}') + + if [ -z "${name}" -o -z "${version}" ]; then + echo "Failed to parse name or version from $(readlink -f ${wheel})" >&2 + exit 1 + fi + + echo "Adding ${name}-${version}..." + + cp ${wheel} . + if [ $? -ne 0 ]; then + echo "Failed to copy $(readlink -f ${wheel})" >&2 + exit 1 + fi + + # Update the upper-constraints file, if necessary + skip_constraint=1 + for skip in ${SKIP_CONSTRAINTS[@]}; do + if [ "${name}" = "${skip}" ]; then + skip_constraint=0 + continue + fi + done + + if [ ${skip_constraint} -eq 0 ]; then + continue + fi + + grep -q "^${name}===${version}\(;.*\)*$" upper-constraints.txt + if [ $? -eq 0 ]; then + # This version already exists in the upper-constraints.txt + continue + fi + + grep -q "^${name}===" upper-constraints.txt + if [ $? -eq 0 ]; then + # Update the version + sed -i "s/^${name}===.*/${name}===${version}/" upper-constraints.txt + else + # Add the module + echo "${name}===${version}" >> upper-constraints.txt + fi +done + +echo "Creating $(basename ${TARBALL_FNAME})..." +tar cf ${TARBALL_FNAME} * +if [ $? -ne 0 ]; then + echo "Failed to create the tarball" >&2 + exit 1 +fi + +echo "Done." + +if [ "${PUSH}" = "yes" ]; then + # + # Push generated wheels tarball to docker registry + # + VERSION=$(increment_version) + + docker import ${TARBALL_FNAME} $REPO_USER/${IMAGE_NAME}:${VERSION} + if [ $? -ne 0 ]; then + echo "Failed command:" >&2 + echo "docker import ${TARBALL_FNAME} $REPO_USER/${IMAGE_NAME}:${VERSION}" >&2 + exit 1 + fi + + docker tag $REPO_USER/${IMAGE_NAME}:${VERSION} $REPO_USER/${IMAGE_NAME}:latest + if [ $? -ne 0 ]; then + echo "Failed command:" >&2 + echo "docker tag $REPO_USER/${IMAGE_NAME}:${VERSION} $REPO_USER/${IMAGE_NAME}:latest" >&2 + exit 1 + fi + + docker push $REPO_USER/${IMAGE_NAME}:${VERSION} + if [ $? -ne 0 ]; then + echo "Failed command:" >&2 + echo "docker push $REPO_USER/${IMAGE_NAME}:${VERSION}" >&2 + exit 1 + fi + + docker push $REPO_USER/${IMAGE_NAME}:latest + if [ $? -ne 0 ]; then + echo "Failed command:" >&2 + echo "docker import ${TARBALL_FNAME} $REPO_USER/${IMAGE_NAME}:${VERSION}" >&2 + exit 1 + fi +fi + +exit 0 + diff --git a/build-tools/build-wheels/docker/centos-dockerfile b/build-tools/build-wheels/docker/centos-dockerfile new file mode 100644 index 00000000..e7e8d7c6 --- /dev/null +++ b/build-tools/build-wheels/docker/centos-dockerfile @@ -0,0 +1,18 @@ +FROM centos:7.5.1804 + +ARG OS_RELEASE=pike + +# Install the necessary packages for building the python modules. +# Some of these are dependencies of the specific modules, and could +# instead be added to the wheels.cfg file in the future. +RUN yum install -y epel-release centos-release-openstack-queens ;\ + yum install -y git gcc zip bzip2 unzip \ + python python-devel python-pip python-wheel \ + wget openldap-devel mariadb mariadb-devel \ + libvirt libvirt-devel liberasurecode-devel nss-devel \ + systemd-devel ;\ + pip install --upgrade pip setuptools + +COPY docker-build-wheel.sh / +COPY ${OS_RELEASE}-wheels.cfg /wheels.cfg + diff --git a/build-tools/build-wheels/docker/docker-build-wheel.sh b/build-tools/build-wheels/docker/docker-build-wheel.sh new file mode 100755 index 00000000..ea02e7d7 --- /dev/null +++ b/build-tools/build-wheels/docker/docker-build-wheel.sh @@ -0,0 +1,301 @@ +#!/bin/bash +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# This utility builds a set of python wheels for upstream packages, +# reading a source list from wheels.cfg +# + +CFGFILE=/wheels.cfg +OUTPUTDIR=/wheels +FAILED_LOG=$OUTPUTDIR/failed.lst + +# +# Function to log the start of a build +# +function startlog { + cat <> $FAILED_LOG + continue + fi + + pushd $basedir + if [ $? -ne 0 ]; then + echo "Failure running: pushd $basedir" + echo $wheelname >> $FAILED_LOG + continue + fi + + git checkout $branch + if [ $? -ne 0 ]; then + echo "Failure running: git checkout $branch" + echo $wheelname >> $FAILED_LOG + continue + fi + + if [ "$fix" == "fix_setup" ]; then + fix_setup + fi + + # Build the wheel + python setup.py bdist_wheel + cp dist/$wheelname $OUTPUTDIR || echo $wheelname >> $FAILED_LOG + popd + done +} + +# +# Function to download a source tarball and build a wheel. +# +function from_tar { + sed 's/#.*//' $CFGFILE | awk -F '|' '$2 == "tar" { print $0; }' | \ + while IFS='|' read wheelname stype wgetsrc basedir fix; do + startlog $wheelname + + if [ -f $OUTPUTDIR/$wheelname ]; then + echo "$wheelname already exists" + continue + fi + + tarball=$(basename $wgetsrc) + if [[ $tarball =~ gz$ ]]; then + taropts="xzf" + elif [[ $tarball =~ bz2$ ]]; then + taropts="xjf" + else + taropts="xf" + fi + + wget $wgetsrc + if [ $? -ne 0 ]; then + echo "Failure running: wget $wgetsrc" + echo $wheelname >> $FAILED_LOG + continue + fi + + tar $taropts $(basename $wgetsrc) + if [ $? -ne 0 ]; then + echo "Failure running: tar $taropts $(basename $wgetsrc)" + echo $wheelname >> $FAILED_LOG + continue + fi + + pushd $basedir + if [ $? -ne 0 ]; then + echo "Failure running: pushd $basedir" + echo $wheelname >> $FAILED_LOG + continue + fi + + if [ "$fix" == "fix_setup" ]; then + fix_setup + fi + + # Build the wheel + python setup.py bdist_wheel + cp dist/$wheelname $OUTPUTDIR || echo $wheelname >> $FAILED_LOG + popd + done +} + +# +# Function to download a source zip file and build a wheel. +# +function from_zip { + sed 's/#.*//' $CFGFILE | awk -F '|' '$2 == "zip" { print $0; }' | \ + while IFS='|' read wheelname stype wgetsrc basedir fix; do + startlog $wheelname + + if [ -f $OUTPUTDIR/$wheelname ]; then + echo "$wheelname already exists" + continue + fi + + wget $wgetsrc + if [ $? -ne 0 ]; then + echo "Failure running: wget $wgetsrc" + echo $wheelname >> $FAILED_LOG + continue + fi + + unzip $(basename $wgetsrc) + if [ $? -ne 0 ]; then + echo "Failure running: unzip $(basename $wgetsrc)" + echo $wheelname >> $FAILED_LOG + continue + fi + + pushd $basedir + if [ $? -ne 0 ]; then + echo "Failure running: pushd $basedir" + echo $wheelname >> $FAILED_LOG + continue + fi + + if [ "$fix" == "fix_setup" ]; then + fix_setup + fi + + # Build the wheel + python setup.py bdist_wheel + cp dist/$wheelname $OUTPUTDIR || echo $wheelname >> $FAILED_LOG + popd + done +} + +# +# Function to download an existing wheel from pypi. +# +function from_pypi { + sed 's/#.*//' $CFGFILE | awk -F '|' '$2 == "pypi" { print $0; }' | \ + while IFS='|' read wheelname stype wgetsrc; do + startlog $wheelname + + if [ -f $OUTPUTDIR/$wheelname ]; then + echo "$wheelname already exists" + continue + fi + + wget $wgetsrc + if [ $? -ne 0 ]; then + echo "Failure running: wget $wgetsrc" + echo $wheelname >> $FAILED_LOG + continue + fi + + cp $wheelname $OUTPUTDIR || echo $wheelname >> $FAILED_LOG + done +} + +rm -f $FAILED_LOG +mkdir -p /build-wheels +cd /build-wheels +from_git +from_tar +from_zip +from_pypi + +if [ -f $FAILED_LOG ]; then + let -i failures=$(cat $FAILED_LOG | wc -l) + + cat <&2 + exit 1 +fi + +OS=centos +OS_RELEASE=pike + +function usage { + cat >&2 < ] [ --release ] + +Options: + --os: Specify base OS (eg. centos) + --release: Openstack release (default: pike) + +EOF +} + +OPTS=$(getopt -o h -l help,os:,release: -- "$@") +if [ $? -ne 0 ]; then + usage + exit 1 +fi + +eval set -- "${OPTS}" + +while true; do + case $1 in + --) + # End of getopt arguments + shift + break + ;; + --os) + OS=$2 + shift 2 + ;; + --release) + OS_RELEASE=$2 + shift 2 + ;; + -h | --help ) + usage + exit 1 + ;; + *) + usage + exit 1 + ;; + esac +done + +# Validate the OS option +SUPPORTED_OS_ARGS=('centos') +VALID_OS=1 +for supported_os in ${SUPPORTED_OS_ARGS[@]}; do + if [ "$OS" = "${supported_os}" ]; then + VALID_OS=0 + break + fi +done +if [ ${VALID_OS} -ne 0 ]; then + echo "Unsupported OS specified: ${OS}" >&2 + echo "Supported OS options: ${SUPPORTED_OS_ARGS[@]}" >&2 + exit 1 +fi + +source ${MY_REPO}/build-tools/git-utils.sh + +function get_wheels_files { + find ${GIT_LIST} -maxdepth 1 -name "${OS}_wheels.inc" +} + +WHEELS_FILES=$(get_wheels_files) +if [ $(echo -n "$WHEELS_FILES" | wc -l) -eq 0 ]; then + echo "Could not find ${OS} wheels.inc files" >&2 + exit 1 +fi + +BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${OS_RELEASE}/stx +if [ -d ${BUILD_OUTPUT_PATH} ]; then + # Wipe out the existing dir to ensure there are no stale files + rm -rf ${BUILD_OUTPUT_PATH} +fi +mkdir -p ${BUILD_OUTPUT_PATH} +cd ${BUILD_OUTPUT_PATH} + +# Extract the wheels +declare -a FAILED +for wheel in $(sed -e 's/#.*//' ${WHEELS_FILES} | sort -u); do + case $OS in + centos) + wheelfile=${MY_WORKSPACE}/std/rpmbuild/RPMS/${wheel}-[^-]*-[^-]*.rpm + + if [ ! -f ${wheelfile} ]; then + echo "Could not find ${wheelfile}" >&2 + FAILED+=($wheel) + continue + fi + + echo Extracting ${wheelfile} + + rpm2cpio ${wheelfile} | cpio -vidu + if [ ${PIPESTATUS[0]} -ne 0 -o ${PIPESTATUS[1]} -ne 0 ]; then + echo "Failed to extract content of ${wheelfile}" >&2 + FAILED+=($wheel) + fi + + ;; + esac +done + +if [ ${#FAILED[@]} -gt 0 ]; then + echo "Failed to find or extract one or more wheel packages:" >&2 + for wheel in ${FAILED[@]}; do + echo "${wheel}" >&2 + done + exit 1 +fi + +exit 0 +