From 2bdac9ac3b09332604919c8e7d39c1676c106ae5 Mon Sep 17 00:00:00 2001 From: Davlet Panech Date: Thu, 8 Apr 2021 15:17:44 -0400 Subject: [PATCH] Build and use py2 wheels with py2 projects Current build process can't handle python2-based projects because we only build wheels based on Ussury (python3). This patch fixes this. - build-wheels/docker/stable-wheels-py2.cfg - build-wheels/docker/dev-wheels-py2.cfg Resurrected original cfg files from commit aa317323716a7f5bb844bb8867eeb8b01313b51e (before upgrade to Ussuri) and saved with -py2 suffix - build-wheels/docker/centos-dockerfile Install python2 packages in addition to python3 - build-wheels/docker/docker-build-wheel.sh * Build with python3 or python2 depending on the environment * Suppress status message at the end because its better handled by the parent script (build-base-wheels.sh) - build-wheels/build-base-wheels.sh * Generate 2 wheel directories, "base" and "base-py2" at each invocation * Use the same builder image and docker script (with different environment) to generate wheels based on different cfg files. Save python3 wheels in "base" and python2 wheels in "base-py2" - build-wheels/build-wheel-tarball.sh * New command-line parameter --python2: generate *-wheels-py2.tar from "base-py2" * Use ussuri or train constraints depending on the presence of --python2 - build-docker-images/build-stx-images.sh * Use *-wheels-py2.tar for python2/loci projects * Fail early on missing --wheels/--wheels-py2 * Fail if multiple projects use the same LABEL * Remove support for config files - build-docker-images/docker-image-build-centos-dev.cfg - build-docker-images/docker-image-build-centos-stable.cfg Removed Change-Id: I2ca444f258a537ed2ba6f68206d32cf59e1802b4 Partial-Bug: 1891416 Signed-off-by: Davlet Panech --- .../build-docker-images/build-stx-images.sh | 251 +++++++----------- .../docker-image-build-centos-dev.cfg | 2 - .../docker-image-build-centos-stable.cfg | 2 - build-tools/build-wheels/build-base-wheels.sh | 200 +++++++++++--- .../build-wheels/build-wheel-tarball.sh | 49 +++- .../build-wheels/docker/centos-dockerfile | 7 + .../build-wheels/docker/dev-wheels-py2.cfg | 18 ++ .../build-wheels/docker/docker-build-wheel.sh | 32 ++- .../build-wheels/docker/stable-wheels-py2.cfg | 173 ++++++++++++ build-tools/build-wheels/openstack.cfg | 13 + 10 files changed, 526 insertions(+), 221 deletions(-) delete mode 100644 build-tools/build-docker-images/docker-image-build-centos-dev.cfg delete mode 100644 build-tools/build-docker-images/docker-image-build-centos-stable.cfg create mode 100644 build-tools/build-wheels/docker/dev-wheels-py2.cfg create mode 100644 build-tools/build-wheels/docker/stable-wheels-py2.cfg create mode 100644 build-tools/build-wheels/openstack.cfg diff --git a/build-tools/build-docker-images/build-stx-images.sh b/build-tools/build-docker-images/build-stx-images.sh index 34494b2e..d80d61f0 100755 --- a/build-tools/build-docker-images/build-stx-images.sh +++ b/build-tools/build-docker-images/build-stx-images.sh @@ -26,7 +26,6 @@ IMAGE_VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp PREFIX=dev LATEST_PREFIX="" PUSH=no -CONFIG_FILE="" HTTP_PROXY="" HTTPS_PROXY="" NO_PROXY="" @@ -34,16 +33,13 @@ DOCKER_USER=${USER} DOCKER_REGISTRY= BASE= WHEELS= -WHEELS_ALTERNATE= -DEFAULT_CONFIG_FILE_DIR="${MY_REPO}/build-tools/build-docker-images" -DEFAULT_CONFIG_FILE_PREFIX="docker-image-build" +WHEELS_PY2= CLEAN=no TAG_LATEST=no TAG_LIST_FILE= TAG_LIST_LATEST_FILE= declare -a ONLY declare -a SKIP -declare -a SERVICES_ALTERNATE declare -i MAX_ATTEMPTS=1 function usage { @@ -52,30 +48,32 @@ Usage: $(basename $0) Options: - --os: Specify base OS (valid options: ${SUPPORTED_OS_ARGS[@]}) - --version: Specify version for output image - --stream: Build stream, stable or dev (default: stable) - --base: Specify base docker image (required option) - --wheels: Specify path to wheels tarball or image, URL or docker tag (required option) - --wheels-alternate: Specify path to alternate wheels tarball or image, URL or docker tag - --push: Push to docker repo - --http_proxy: Set proxy :, urls splitted with "," - --https_proxy: Set proxy :, urls splitted with "," - --no_proxy: Set proxy , urls splitted with "," - --user: Docker repo userid - --registry: Docker registry - --prefix: Prefix on the image tag (default: dev) - --latest: Add a 'latest' tag when pushing + --os: Specify base OS (valid options: ${SUPPORTED_OS_ARGS[@]}) + --version: Specify version for output image + --stream: Build stream, stable or dev (default: stable) + --base: Specify base docker image (required option) + --wheels: Specify path to wheels tarball or image, URL or docker tag + (required when building loci projects) + --wheels-py2: Use this wheels tarball for Python2 projects + (default: work out from --wheels) + --wheels-alternate: same as --wheels-py2 + --push: Push to docker repo + --http_proxy: Set proxy :, urls splitted with "," + --https_proxy: Set proxy :, urls splitted with "," + --no_proxy: Set proxy , urls splitted with "," + --user: Docker repo userid + --registry: Docker registry + --prefix: Prefix on the image tag (default: dev) + --latest: Add a 'latest' tag when pushing --latest-prefix: Alternative prefix on the latest image tag - --clean: Remove image(s) from local registry + --clean: Remove image(s) from local registry --only : Only build the specified image(s). Multiple images can be specified with a comma-separated list, or with multiple --only arguments. --skip : Skip building the specified image(s). Multiple images can be specified with a comma-separated list, or with multiple --skip arguments. - --attempts: Max attempts, in case of failure (default: 1) - --config-file:Specify a path to a config file which will specify additional arguments to be passed into the the command + --attempts: Max attempts, in case of failure (default: 1) EOF @@ -97,63 +95,6 @@ function is_empty { test $# -eq 0 } -function get_args_from_file { - # get additional build args from specified file. - local -a config_items - - echo "Get args from file: $1" - for i in $(cat $1) - do - config_items=($(echo $i | sed s/=/\ /g)) - echo "--${config_items[0]} ${config_items[1]}" - case ${config_items[0]} in - base) - if [ -z "${BASE}" ]; then - BASE=${config_items[1]} - fi - ;; - user) - if [ -z "${DOCKER_USER}" ]; then - DOCKER_USER=${config_items[1]} - fi - ;; - proxy) - if [ -z "${PROXY}" ]; then - PROXY=${config_items[1]} - fi - ;; - registry) - if [ -z "${DOCKER_REGISTRY}" ]; then - # Add a trailing / if needed - DOCKER_REGISTRY="${config_items[1]%/}/" - fi - ;; - only) - # Read comma-separated values into array - if [ -z "${ONLY}" ]; then - # Read comma-separated values into array - ONLY=(`echo ${config_items[1]} | sed s/,/\ /g`) - fi - ;; - wheels) - if [ -z "${WHEELS}" ]; then - WHEELS=${config_items[1]} - fi - ;; - wheels_alternate) - if [ -z "${WHEELS_ALTERNATE}" ]; then - WHEELS_ALTERNATE=${config_items[1]} - echo "WHEELS_ALTERNATE: ${WHEELS_ALTERNATE}" >&2 - fi - ;; - services_alternate) - SERVICES_ALTERNATE=(`echo ${config_items[1]} | sed s/,/\ /g`) - echo "SERVICES_ALTERNATE: ${SERVICES_ALTERNATE[@]}" >&2 - ;; - esac - done -} - # # get_git: Clones a git into a subdirectory of ${WORKDIR}, and # leaves you in that directory. On error the directory @@ -394,16 +335,6 @@ function build_image_loci { local PYTHON3 PYTHON3=$(source ${image_build_file} && echo ${PYTHON3}) - if is_in ${PROJECT} ${SKIP[@]} || is_in ${LABEL} ${SKIP[@]}; then - echo "Skipping ${LABEL}" - return 0 - fi - - if ! is_empty ${ONLY[@]} && ! is_in ${PROJECT} ${ONLY[@]} && ! is_in ${LABEL} ${ONLY[@]}; then - echo "Skipping ${LABEL}" - return 0 - fi - echo "Building ${LABEL}" local -a BUILD_ARGS= @@ -411,9 +342,9 @@ function build_image_loci { BUILD_ARGS+=(--build-arg PROJECT_REPO=${PROJECT_REPO}) BUILD_ARGS+=(--build-arg FROM=${BASE}) - if is_in ${LABEL} ${SERVICES_ALTERNATE[@]}; then + if [ "${PYTHON3}" != "yes" ] ; then echo "Python2 service ${LABEL}" - BUILD_ARGS+=(--build-arg WHEELS=${WHEELS_ALTERNATE}) + BUILD_ARGS+=(--build-arg WHEELS=${WHEELS_PY2}) else echo "Python3 service ${LABEL}" BUILD_ARGS+=(--build-arg WHEELS=${WHEELS}) @@ -518,16 +449,6 @@ function build_image_docker { local DOCKER_PATCHES DOCKER_PATCHES=$(source ${image_build_file} && for p in ${DOCKER_PATCHES}; do echo $(dirname ${image_build_file})/${p}; done) - if is_in ${PROJECT} ${SKIP[@]} || is_in ${LABEL} ${SKIP[@]}; then - echo "Skipping ${LABEL}" - return 0 - fi - - if ! is_empty ${ONLY[@]} && ! is_in ${PROJECT} ${ONLY[@]} && ! is_in ${LABEL} ${ONLY[@]}; then - echo "Skipping ${LABEL}" - return 0 - fi - echo "Building ${LABEL}" local real_docker_context @@ -625,16 +546,6 @@ function build_image_script { local SOURCE_PATCHES SOURCE_PATCHES=$(source ${image_build_file} && for p in ${SOURCE_PATCHES}; do echo $(dirname ${image_build_file})/${p}; done) - if is_in ${PROJECT} ${SKIP[@]} || is_in ${LABEL} ${SKIP[@]}; then - echo "Skipping ${LABEL}" - return 0 - fi - - if ! is_empty ${ONLY[@]} && ! is_in ${PROJECT} ${ONLY[@]} && ! is_in ${LABEL} ${ONLY[@]}; then - echo "Skipping ${LABEL}" - return 0 - fi - # Validate the COMMAND option SUPPORTED_COMMAND_ARGS=('bash') local VALID_COMMAND=1 @@ -715,7 +626,7 @@ function build_image { esac } -OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,http_proxy:,https_proxy:,no_proxy:,user:,registry:,base:,wheels:,wheels-alternate:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts:,config-file: -- "$@") +OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,http_proxy:,https_proxy:,no_proxy:,user:,registry:,base:,wheels:,wheels-alternate:,wheels-py2:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -742,8 +653,8 @@ while true; do WHEELS=$2 shift 2 ;; - --wheels-alternate) - WHEELS_ALTERNATE=$2 + --wheels-alternate|--wheels-py2) + WHEELS_PY2=$2 shift 2 ;; --version) @@ -813,10 +724,6 @@ while true; do MAX_ATTEMPTS=$2 shift 2 ;; - --config-file) - CONFIG_FILE=$2 - shift 2 - ;; -h | --help ) usage exit 1 @@ -842,37 +749,80 @@ if [ ${VALID_OS} -ne 0 ]; then exit 1 fi -DEFAULT_CONFIG_FILE="${DEFAULT_CONFIG_FILE_DIR}/${DEFAULT_CONFIG_FILE_PREFIX}-${OS}-${BUILD_STREAM}.cfg" - -# Read additional arguments from config file if it exists. -if [[ -z "$CONFIG_FILE" ]] && [[ -f ${DEFAULT_CONFIG_FILE} ]]; then - CONFIG_FILE=${DEFAULT_CONFIG_FILE} -fi -if [[ ! -z ${CONFIG_FILE} ]]; then - if [[ -f ${CONFIG_FILE} ]]; then - get_args_from_file ${CONFIG_FILE} - else - echo "Config file not found: ${CONFIG_FILE}" - exit 1 - fi -fi - -if [ -z "${WHEELS}" ]; then - echo "Path to wheels tarball must be specified with --wheels option." >&2 - exit 1 -fi - -if [ ${#SERVICES_ALTERNATE[@]} -ne 0 ] && [ -z "${WHEELS_ALTERNATE}" ]; then - echo "Path to wheels-alternate tarball must be specified with --wheels-alternate option"\ - "if python2 based services need to be build!" >&2 - exit 1 -fi - if [ -z "${BASE}" ]; then echo "Base image must be specified with --base option." >&2 exit 1 fi +# Guess WHEELS_PY2 if missing +if [[ -z "$WHEELS_PY2" && -n "$WHEELS" ]]; then + # http://foo/bar.tar?xxx#yyy => http://foo/bar-py2.tar?xxx#yyy + WHEELS_PY2="$(echo "$WHEELS" | sed -r 's,^([^#?]*)(\.tar)(\.gz|\.bz2|\.xz)?([#?].*)?$,\1-py2\2\3\4,i')" + if [[ "$WHEELS" == "$WHEELS_PY2" ]]; then + echo "Unable to guess --wheels-py2, please specify it explicitly" >&2 + exit 1 + fi +fi + +# Find the directives files +IMAGE_BUILD_FILES=() +function find_image_build_files { + local image_build_inc_file image_build_dir image_build_file + local -A all_labels + + for image_build_inc_file in $(find ${GIT_LIST} -maxdepth 1 -name "${OS}_${BUILD_STREAM}_docker_images.inc"); do + basedir=$(dirname ${image_build_inc_file}) + for image_build_dir in $(sed -e 's/#.*//' ${image_build_inc_file} | sort -u); do + for image_build_file in ${basedir}/${image_build_dir}/${OS}/*.${BUILD_STREAM}_docker_image; do + + # reset & read image build directive vars + local BUILDER= + local PROJECT= + local LABEL= + local PYTHON3= + PROJECT="$(source ${image_build_file} && echo ${PROJECT})" + BUILDER="$(source ${image_build_file} && echo ${BUILDER})" + LABEL="$(source ${image_build_file} && echo ${LABEL})" + PYTHON3="$(source ${image_build_file} && echo ${PYTHON3})" + + # make sure labels are unique + if [[ -n "${all_labels["$LABEL"]}" ]] ; then + echo "The following files define the same LABEL $LABEL" >&2 + echo " ${all_labels["$LABEL"]}" >&2 + echo " ${image_build_file}" >&2 + exit 1 + fi + all_labels["$LABEL"]="$image_build_file" + + # skip images we don't want to build + if is_in ${PROJECT} ${SKIP[@]} || is_in ${LABEL} ${SKIP[@]}; then + continue + fi + if ! is_empty ${ONLY[@]} && ! is_in ${PROJECT} ${ONLY[@]} && ! is_in ${LABEL} ${ONLY[@]}; then + continue + fi + + # loci builders require a wheels tarball + if [[ "${BUILDER}" == "loci" ]] ; then + # python3 projects require $WHEELS + if [[ "${PYTHON3}" == "yes" && -z "${WHEELS}" ]] ; then + echo "You are building python3 services with loci, but you didn't specify --wheels!" >&2 + exit 1 + # python2 projects require WHEELS_PY2 + elif [[ "${PYTHON3}" != "yes" && -z "${WHEELS_PY2}" ]] ; then + echo "You are building python2 services with loci, but you didn't specify --wheels-py2!" >&2 + exit 1 + fi + fi + + # Save image build file in the global list + IMAGE_BUILD_FILES+=("$image_build_file") + done + done + done +} +find_image_build_files + IMAGE_TAG="${OS}-${BUILD_STREAM}" IMAGE_TAG_LATEST="${IMAGE_TAG}-latest" @@ -933,15 +883,10 @@ if ! (grep -q rh-python36-mod_wsgi ${WORKDIR}/loci/bindep.txt); then echo 'rh-python36-mod_wsgi [platform:rpm !platform:suse (apache python3)]' >> ${WORKDIR}/loci/bindep.txt fi -# Find the directives files -for image_build_inc_file in $(find ${GIT_LIST} -maxdepth 1 -name "${OS}_${BUILD_STREAM}_docker_images.inc"); do - basedir=$(dirname ${image_build_inc_file}) - for image_build_dir in $(sed -e 's/#.*//' ${image_build_inc_file} | sort -u); do - for image_build_file in ${basedir}/${image_build_dir}/${OS}/*.${BUILD_STREAM}_docker_image; do - # Failures are reported by the build functions - build_image ${image_build_file} - done - done +# Build everything +for image_build_file in "${IMAGE_BUILD_FILES[@]}" ; do + # Failures are reported by the build functions + build_image ${image_build_file} done if [ "${CLEAN}" = "yes" -a ${#RESULTS_BUILT[@]} -gt 0 ]; then diff --git a/build-tools/build-docker-images/docker-image-build-centos-dev.cfg b/build-tools/build-docker-images/docker-image-build-centos-dev.cfg deleted file mode 100644 index 80e5ace3..00000000 --- a/build-tools/build-docker-images/docker-image-build-centos-dev.cfg +++ /dev/null @@ -1,2 +0,0 @@ -services_alternate=stx-fm-rest-api,stx-keystone-api-proxy,stx-nova-api-proxy,stx-platformclients -wheels_alternate=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/stx-centos-py2_dev-wheels.tar diff --git a/build-tools/build-docker-images/docker-image-build-centos-stable.cfg b/build-tools/build-docker-images/docker-image-build-centos-stable.cfg deleted file mode 100644 index 871d2271..00000000 --- a/build-tools/build-docker-images/docker-image-build-centos-stable.cfg +++ /dev/null @@ -1,2 +0,0 @@ -services_alternate=stx-fm-rest-api,stx-keystone-api-proxy,stx-nova-api-proxy,stx-platformclients -wheels_alternate=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/stx-centos-py2_stable-wheels.tar diff --git a/build-tools/build-wheels/build-base-wheels.sh b/build-tools/build-wheels/build-base-wheels.sh index 4ab913c1..b0274f22 100755 --- a/build-tools/build-wheels/build-base-wheels.sh +++ b/build-tools/build-wheels/build-base-wheels.sh @@ -114,8 +114,6 @@ while true; do esac done -BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${BUILD_STREAM}/base - BUILD_IMAGE_NAME="${USER}-$(basename ${MY_WORKSPACE})-wheelbuilder:${OS}-${BUILD_STREAM}" # BUILD_IMAGE_NAME can't have caps if it's passed to docker build -t $BUILD_IMAGE_NAME. @@ -123,7 +121,6 @@ BUILD_IMAGE_NAME="${USER}-$(basename ${MY_WORKSPACE})-wheelbuilder:${OS}-${BUILD BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME,,}" DOCKER_FILE=${DOCKER_PATH}/${OS}-dockerfile -WHEELS_CFG=${DOCKER_PATH}/${BUILD_STREAM}-wheels.cfg function supported_os_list { for f in ${DOCKER_PATH}/*-dockerfile; do @@ -137,40 +134,75 @@ if [ ! -f ${DOCKER_FILE} ]; then exit 1 fi -if [ ! -f ${WHEELS_CFG} ]; then - echo "Required file does not exist: ${WHEELS_CFG}" >&2 - exit 1 -fi +# Print a loud message +function notice { + ( + set +x + echo + echo ====================================== + for s in "$@" ; do + echo "$s" + done + echo ====================================== + echo + ) 2>&1 +} -# -# Check build output directory for unexpected files, -# ie. wheels from old builds that are no longer in wheels.cfg -# -if [ -d ${BUILD_OUTPUT_PATH} ]; then +# prefix each line of a command's output +# also redirects command's STDERR to STDOUT +log_prefix() { + local prefix="$1" ; shift + "$@" 2>&1 | awk -v prefix="$prefix" '{print prefix $0}' + # return false if the command (rather than awk) failed + [ ${PIPESTATUS[0]} -eq 0 ] +} - for f in ${BUILD_OUTPUT_PATH}/*; do - grep -q "^$(basename $f)|" ${WHEELS_CFG} - if [ $? -ne 0 ]; then - echo "Deleting stale file: $f" - rm -f $f - fi - done -else - mkdir -p ${BUILD_OUTPUT_PATH} - if [ $? -ne 0 ]; then - echo "Failed to create directory: ${BUILD_OUTPUT_PATH}" >&2 + +# Make sure a file exists, exit otherwise +function require_file { + if [ ! -f "${1}" ]; then + echo "Required file does not exist: ${1}" >&2 exit 1 fi -fi +} -# Check to see if we need to build anything -BUILD_NEEDED=no -for wheel in $(cat ${WHEELS_CFG} | sed 's/#.*//' | awk -F '|' '{print $1}'); do - if [[ "${wheel}" =~ \* || ! -f ${BUILD_OUTPUT_PATH}/${wheel} ]]; then - BUILD_NEEDED=yes - break +# Check build output directory for unexpected files, +# ie. wheels from old builds that are no longer in wheels.cfg +function prepare_output_dir { + local output_dir="$1" + local wheels_cfg="$2" + if [ -d ${output_dir} ]; then + local f + for f in ${output_dir}/*; do + if [ -f $f ] ; then + grep -q "^$(basename $f)|" ${wheels_cfg} + if [ $? -ne 0 ]; then + echo "Deleting stale file: $f" + rm -f $f + fi + fi + done + else + mkdir -p ${output_dir} + if [ $? -ne 0 ]; then + echo "Failed to create directory: ${output_dir}" >&2 + exit 1 + fi fi -done +} + +BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${BUILD_STREAM}/base +BUILD_OUTPUT_PATH_PY2=${MY_WORKSPACE}/std/build-wheels-${OS}-${BUILD_STREAM}/base-py2 +WHEELS_CFG=${DOCKER_PATH}/${BUILD_STREAM}-wheels.cfg +WHEELS_CFG_PY2=${DOCKER_PATH}/${BUILD_STREAM}-wheels-py2.cfg + +# make sure .cfg files exist +require_file "${WHEELS_CFG}" +require_file "${WHEELS_CFG_PY2}" + +# prepare output directories +prepare_output_dir "${BUILD_OUTPUT_PATH}" "${WHEELS_CFG}" +prepare_output_dir "${BUILD_OUTPUT_PATH_PY2}" "${WHEELS_CFG_PY2}" if [ "${BUILD_STREAM}" = "dev" -o "${BUILD_STREAM}" = "master" ]; then # Download the master wheel from loci, so we're only building pieces not covered by it @@ -194,16 +226,30 @@ if [ "${BUILD_STREAM}" = "dev" -o "${BUILD_STREAM}" = "master" ]; then docker run --name ${USER}_inspect_wheels ${MASTER_WHEELS_IMAGE} noop 2>/dev/null echo "Extracting wheels from ${MASTER_WHEELS_IMAGE}" - docker export ${USER}_inspect_wheels | tar x -C ${BUILD_OUTPUT_PATH} '*.whl' + rm -rf "${BUILD_OUTPUT_PATH}-loci" + mkdir -p "$BUILD_OUTPUT_PATH-loci" + docker export ${USER}_inspect_wheels | tar x -C "${BUILD_OUTPUT_PATH}-loci" '*.whl' if [ ${PIPESTATUS[0]} -ne 0 -o ${PIPESTATUS[1]} -ne 0 ]; then echo "Failed to extract wheels from ${MASTER_WHEELS_IMAGE}" >&2 docker rm ${USER}_inspect_wheels if [ ${MASTER_WHEELS_PRESENT} -ne 0 ]; then docker image rm ${MASTER_WHEELS_IMAGE} fi + rm -rf "${BUILD_OUTPUT_PATH}-loci" exit 1 fi + # copy loci wheels in base and base-py2 directories + if ! cp "${BUILD_OUTPUT_PATH}-loci"/*.whl "${BUILD_OUTPUT_PATH}"/ ; then + echo "Failed to copy wheels to ${BUILD_OPUTPUT_PATH}" >&2 + exit 1 + fi + if ! cp "${BUILD_OUTPUT_PATH}-loci"/*.whl "${BUILD_OUTPUT_PATH_PY2}"/ ; then + echo "Failed to copy wheels to ${BUILD_OPUTPUT_PATH_PY2}" >&2 + exit 1 + fi + rm -rf "${BUILD_OUTPUT_PATH}-loci" + docker rm ${USER}_inspect_wheels if [ ${MASTER_WHEELS_PRESENT} -ne 0 ]; then @@ -211,7 +257,21 @@ if [ "${BUILD_STREAM}" = "dev" -o "${BUILD_STREAM}" = "master" ]; then fi fi -if [ "${BUILD_NEEDED}" = "no" ]; then +# check if there are any wheels missing +function all_wheels_exist { + local output_dir="$1" + local wheels_cfg="$2" + local wheel + for wheel in $(cat "${wheels_cfg}" | sed 's/#.*//' | awk -F '|' '{print $1}'); do + if [[ "${wheel}" =~ \* || ! -f ${output_dir}/${wheel} ]]; then + return 1 + fi + done + return 0 +} + +if all_wheels_exist "${BUILD_OUTPUT_PATH}" "${WHEELS_CFG}" && \ + all_wheels_exist "${BUILD_OUTPUT_PATH_PY2}" "${WHEELS_CFG_PY2}" ; then echo "All base wheels are already present. Skipping build." exit 0 fi @@ -247,12 +307,10 @@ if [ $? -ne 0 ]; then fi # Run the image, executing the build-wheel.sh script -RM_OPT= -if [ "${KEEP_CONTAINER}" = "no" ]; then - RM_OPT="--rm" -fi - declare -a RUN_ARGS +if [ "${KEEP_CONTAINER}" = "no" ]; then + RUN_ARGS+=(--rm) +fi if [ ! -z "$HTTP_PROXY" ]; then RUN_ARGS+=(--env http_proxy=$HTTP_PROXY) fi @@ -262,11 +320,23 @@ fi if [ ! -z "$NO_PROXY" ]; then RUN_ARGS+=(--env no_proxy=$NO_PROXY) fi - -RUN_ARGS+=(${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh) +RUN_ARGS+=(--env DISPLAY_RESULT=no) # Run container to build wheels -with_retries ${MAX_ATTEMPTS} docker run ${RUN_ARGS[@]} +rm -f ${BUILD_OUTPUT_PATH}/failed.lst +rm -f ${BUILD_OUTPUT_PATH_PY2}/failed.lst + +notice "building python3 wheels" +log_prefix "[python3] " \ + with_retries ${MAX_ATTEMPTS} \ + docker run ${RUN_ARGS[@]} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh +BUILD_STATUS=$? + +notice "building python2 wheels" +log_prefix "[python2] " \ + with_retries ${MAX_ATTEMPTS} \ + docker run ${RUN_ARGS[@]} -v ${BUILD_OUTPUT_PATH_PY2}:/wheels --env PYTHON=python2 ${BUILD_IMAGE_NAME} /docker-build-wheel.sh +BUILD_STATUS_PY2=$? if [ "${KEEP_IMAGE}" = "no" ]; then # Delete the builder image @@ -287,8 +357,52 @@ if [ "${KEEP_IMAGE}" = "no" ]; then fi # Check for failures -if [ -f ${BUILD_OUTPUT_PATH}/failed.lst ]; then - # Failures would already have been reported +check_result() { + local python="$1" + local status="$2" + local dir="$3" + + # There's a failed images list + if [ -f "${dir}/failed.lst" ]; then + let failures=$(cat "${dir}/failed.lst" | wc -l) + + cat <&2 <&2 exit 1 fi -with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt +with_retries ${MAX_ATTEMPTS} wget "${OPENSTACK_REQ_URL}/upper-constraints.txt" if [ $? -ne 0 ]; then echo "Failed to download upper-constraints.txt" >&2 exit 1 @@ -230,7 +259,7 @@ done shopt -s nullglob # Copy the base and stx wheels, updating upper-constraints.txt as necessary -for wheel in ../base/*.whl ../stx/wheels/*.whl; do +for wheel in ../base${PY_SUFFIX}/*.whl ../stx/wheels/*.whl; do # Get the wheel name and version from the METADATA METADATA=$(unzip -p ${wheel} '*/METADATA') name=$(echo "${METADATA}" | grep '^Name:' | awk '{print $2}') diff --git a/build-tools/build-wheels/docker/centos-dockerfile b/build-tools/build-wheels/docker/centos-dockerfile index c2dd825f..170a71a9 100644 --- a/build-tools/build-wheels/docker/centos-dockerfile +++ b/build-tools/build-wheels/docker/centos-dockerfile @@ -20,3 +20,10 @@ RUN set -ex ;\ COPY docker-build-wheel.sh / COPY ${BUILD_STREAM}-wheels.cfg /wheels.cfg +# Python2 packages +RUN set -ex; \ + yum -y install python python-devel ;\ + wget https://bootstrap.pypa.io/pip/2.7/get-pip.py ;\ + python get-pip.py +COPY ${BUILD_STREAM}-wheels-py2.cfg /wheels-py2.cfg + diff --git a/build-tools/build-wheels/docker/dev-wheels-py2.cfg b/build-tools/build-wheels/docker/dev-wheels-py2.cfg new file mode 100644 index 00000000..0bf19816 --- /dev/null +++ b/build-tools/build-wheels/docker/dev-wheels-py2.cfg @@ -0,0 +1,18 @@ +# +# git: wheelname|git|git-source|basedir|branch +# tar: wheelname|tar|wget-source|basedir +# pypi: wheelname|pypi|wget-source +# zip: wheelname|zip|wget-source|basedir +# +# If fix_setup must be called, add |fix_setup at the end of the line +# +amqplib-1.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2 +lz4-0.9.0-cp27-none-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0 +panko-5.0.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a9/89/d666e0889d869e41c9b7f87a0a34858b2520782b82e025da84c98e0db8f6/panko-5.0.0.tar.gz|panko-5.0.0 +google_api_python_client-1.7.7-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/d7/47/940908e52487440f61fb93ad55cbbe3a28235d3bb143b26affb17b37dd28/google_api_python_client-1.7.7-py2.py3-none-any.whl +neutron_lib-*.whl|git|https://github.com/openstack/neutron-lib|neutron-lib|master +python_openstackclient-*.whl|git|https://github.com/openstack/python-openstackclient|python-openstackclient|master +openstacksdk-*.whl|git|https://github.com/openstack/openstacksdk|openstacksdk|master +networking_sfc-8.0.0.0b2-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/6a/a8/0e9bdd1f87dfb50682f23a01f590530ec8fa715e51127cf9f58d1905886c/networking_sfc-8.0.0.0b2-py2.py3-none-any.whl +croniter-0.3.29-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/a9/c9/11182a2507798c661b04a7914739ea8ca73a738e6869a23742029f51bc1a/croniter-0.3.29-py2.py3-none-any.whl +pecan-1.3.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/93/98/889d7615595e894f4f7e4c17d4008c822c8e39e650c8ab390cc6c39b99c4/pecan-1.3.3.tar.gz|pecan-1.3.3 diff --git a/build-tools/build-wheels/docker/docker-build-wheel.sh b/build-tools/build-wheels/docker/docker-build-wheel.sh index 9da8b84f..59e752a4 100755 --- a/build-tools/build-wheels/docker/docker-build-wheel.sh +++ b/build-tools/build-wheels/docker/docker-build-wheel.sh @@ -10,8 +10,14 @@ CFGFILE=/wheels.cfg OUTPUTDIR=/wheels -FAILED_LOG=$OUTPUTDIR/failed.lst +FAILED_LOG="${OUTPUTDIR}/failed.lst" +: ${DISPLAY_RESULT=yes} declare -i MAX_ATTEMPTS=5 +: ${PYTHON=python3} +if [[ "${PYTHON}" == "python2" ]] ; then + CFGFILE=/wheels-py2.cfg + FAILED_LOG="${OUTPUTDIR}/failed-py2.lst" +fi # # Function to log the start of a build @@ -184,7 +190,7 @@ function from_git { fi # Build the wheel - python3 setup.py bdist_wheel + ${PYTHON} setup.py bdist_wheel if [ -f dist/$wheelname ]; then cp dist/$wheelname $OUTPUTDIR || echo $wheelname >> $FAILED_LOG else @@ -244,7 +250,7 @@ function from_tar { fi # Build the wheel - python3 setup.py bdist_wheel + ${PYTHON} setup.py bdist_wheel if [ -f dist/$wheelname ]; then cp dist/$wheelname $OUTPUTDIR || echo $wheelname >> $FAILED_LOG else @@ -295,7 +301,7 @@ function from_zip { fi # Build the wheel - python3 setup.py bdist_wheel + ${PYTHON} setup.py bdist_wheel if [ -f dist/$wheelname ]; then cp dist/$wheelname $OUTPUTDIR || echo $wheelname >> $FAILED_LOG else @@ -339,24 +345,28 @@ from_tar from_zip from_pypi -if [ -f $FAILED_LOG ]; then - let failures=$(cat $FAILED_LOG | wc -l) +if [ -f "${FAILED_LOG}" ]; then + if [ "${DISPLAY_RESULT}" = yes ] ; then + let failures=$(cat "${FAILED_LOG}" | wc -l) - cat <