From a0ca6ce157b8a84f282d8abbdd27c75f48d991ff Mon Sep 17 00:00:00 2001 From: Mathieu Mitchell Date: Mon, 22 Aug 2016 15:43:39 -0400 Subject: [PATCH] Enforce upper-constraints when building ramdisks Currently, building ramdisks installs ironic-python-agent without any upper-constraints. This causes the package to be installed with newer, untested dependencies. This commits introduces a tool to generate a local upper-constraints file based on predefined strategies (below). Additionally, the fallback to the openstack/requirements uses the URL defined in tox.ini instead of redefining it. This prevents having to keep track of two separate variables when releasing. upper-constraints lookup strategies (in order): * UPPER_CONSTRAINTS_FILE points to a local file * UPPER_CONSTRAINTS_FILE points to a URL * /opt/stack/new/requirements/upper-constraints.txt * upper-constraints.txt from openstack/requirements git repository Partial-bug: #1616554 Change-Id: Ib5c0c57cafdb6ffd7456e61f3b1bb5fa57520e5a --- .gitignore | 3 + Dockerfile | 5 +- .../extract_upper_constraints_from_tox_ini.sh | 9 +++ .../common/generate_upper_constraints.sh | 79 +++++++++++++++++++ imagebuild/coreos/docker_build.bash | 6 +- imagebuild/tinyipa/build-tinyipa.sh | 11 ++- imagebuild/tinyipa/finalise-tinyipa.sh | 4 +- 7 files changed, 108 insertions(+), 9 deletions(-) create mode 100755 imagebuild/common/extract_upper_constraints_from_tox_ini.sh create mode 100755 imagebuild/common/generate_upper_constraints.sh diff --git a/.gitignore b/.gitignore index 166a1505f..f7e20d760 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ doc/build # release notes build releasenotes/build + +# upper-constraints handling for image builds +upper-constraints.txt diff --git a/Dockerfile b/Dockerfile index 1857efb9b..8c7a56a68 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,11 +39,10 @@ RUN apt-mark manual python-minimal # Install requirements separately, because pip understands a git+https url # while setuptools doesn't RUN proxy.sh pip install --upgrade pip -# TODO(jroll) use upper-constraints here -RUN proxy.sh pip install --no-cache-dir -r /tmp/ironic-python-agent/requirements.txt +RUN proxy.sh pip install -c /tmp/ironic-python-agent/upper-constraints.txt --no-cache-dir -r /tmp/ironic-python-agent/requirements.txt # This will succeed because all the dependencies were installed previously -RUN proxy.sh pip install --no-cache-dir /tmp/ironic-python-agent +RUN proxy.sh pip install -c /tmp/ironic-python-agent/upper-constraints.txt --no-cache-dir /tmp/ironic-python-agent # Remove no longer needed packages # NOTE(jroll) leave git to avoid strange apt issues in downstream Dockerfiles diff --git a/imagebuild/common/extract_upper_constraints_from_tox_ini.sh b/imagebuild/common/extract_upper_constraints_from_tox_ini.sh new file mode 100755 index 000000000..86bd99733 --- /dev/null +++ b/imagebuild/common/extract_upper_constraints_from_tox_ini.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# NOTE(mmitchell): This extracts the URL defined as the default value for +# UPPER_CONSTRAINTS_FILE in tox.ini. This is used by image +# builders to avoid duplicating the default value in multiple +# scripts. This is specially done to leverage the release +# tools that automatically update the tox.ini when projects +# are released. +sed -n 's/^.*{env:UPPER_CONSTRAINTS_FILE\:\([^}]*\)}.*$/\1/p' $1 | head -n1 + diff --git a/imagebuild/common/generate_upper_constraints.sh b/imagebuild/common/generate_upper_constraints.sh new file mode 100755 index 000000000..cd61b784b --- /dev/null +++ b/imagebuild/common/generate_upper_constraints.sh @@ -0,0 +1,79 @@ +#!/bin/bash -eu + +SCRIPT_NAME=$(basename $0) +COMMON_ROOT=$(dirname $0) +DESTINATION="$1" +TOX_INI_UPPER_CONSTRAINT_URL="$(${COMMON_ROOT}/extract_upper_constraints_from_tox_ini.sh ${COMMON_ROOT}/../../tox.ini)" + +copy() { + local src=$1 + local destination=$2 + + if test -z "${src}"; then + return 1 + fi + + if test -e "${src}"; then + log "File '${src}' exists. Using as upper-constraints." + cp "${src}" "${destination}" + else + log "File '${src}' not found. Skipping local file strategy." + return 1 + fi + return 0 +} + +download() { + local url=$1 + local destination=$2 + + if test -z "${url}"; then + return 1 + else + log "Downloading from '${url}'" + curl ${url} -o "${destination}" + fi + return 0 +} + +log() { + echo "${SCRIPT_NAME}: ${@}" +} + +fail() { + log ${@} + exit 1 +} + +upper_constraints_is_not_null() { + test "${UPPER_CONSTRAINTS_FILE:-""}" != "" +} + +copy_uc() { + copy "${UPPER_CONSTRAINTS_FILE:-""}" "${DESTINATION}" +} + +download_uc() { + download "${UPPER_CONSTRAINTS_FILE:-""}" "${DESTINATION}" +} + +copy_new_requirements_uc() { + copy "/opt/stack/new/requirements/upper-constraints.txt" "${DESTINATION}" +} + +download_from_tox_ini_url() { + log "tox.ini indicates '${TOX_INI_UPPER_CONSTRAINT_URL}' as fallback." + download "${TOX_INI_UPPER_CONSTRAINT_URL}" "${DESTINATION}" +} + +log "Generating local constraints file..." + +if upper_constraints_is_not_null; then + log "UPPER_CONSTRAINTS_FILE is defined as '${UPPER_CONSTRAINTS_FILE:-""}'" + copy_uc || download_uc || fail "Failed to copy or download file indicated in UPPER_CONSTRAINTS_FILE." +else + log "UPPER_CONSTRAINTS_FILE is not defined. Using fallback strategies." + + copy_new_requirements_uc || \ + download_from_tox_ini_url || fail "Failed to download upper-constraints.txt from '${TOX_INI_UPPER_CONSTRAINT_URL}'." +fi diff --git a/imagebuild/coreos/docker_build.bash b/imagebuild/coreos/docker_build.bash index 9386ef677..5f503d37a 100755 --- a/imagebuild/coreos/docker_build.bash +++ b/imagebuild/coreos/docker_build.bash @@ -7,6 +7,7 @@ set -e OUTPUT_FILE="oem/container.tar.gz" +IPA_ROOT=$(readlink -f $(dirname $0)/../../) # If there's already a container.tar.gz, don't overwrite it -- instead, bail if [[ -e "${OUTPUT_FILE}" ]]; then @@ -15,7 +16,10 @@ if [[ -e "${OUTPUT_FILE}" ]]; then fi # Build the docker image -cd ../../ +# Everything from ${IPA_ROOT} will be available under /tmp/ironic-python-agent in Docker +cd ${IPA_ROOT} + +imagebuild/common/generate_upper_constraints.sh ${IPA_ROOT}/upper-constraints.txt # TODO(jlvilla): Once Docker 1.9 is widely deployed, switch to using the 'ARG' # command which was added in Docker 1.9. Currently Ubuntu 14.04 uses Docker diff --git a/imagebuild/tinyipa/build-tinyipa.sh b/imagebuild/tinyipa/build-tinyipa.sh index 0c6059537..33dae7b4c 100755 --- a/imagebuild/tinyipa/build-tinyipa.sh +++ b/imagebuild/tinyipa/build-tinyipa.sh @@ -65,6 +65,9 @@ cd ../.. rm -rf *.egg-info python setup.py sdist --dist-dir "$BUILDDIR/tmp/localpip" --quiet cp requirements.txt $BUILDDIR/tmp/ipa-requirements.txt + +imagebuild/common/generate_upper_constraints.sh upper-constraints.txt +cp upper-constraints.txt $BUILDDIR/tmp/upper-constraints.txt cd $WORKDIR sudo cp /etc/resolv.conf $BUILDDIR/etc/resolv.conf @@ -88,10 +91,10 @@ done < $WORKDIR/build_files/buildreqs.lst # Build python wheels $CHROOT_CMD python /tmp/get-pip.py $CHROOT_CMD pip install pbr -$CHROOT_CMD pip wheel --wheel-dir /tmp/wheels setuptools -$CHROOT_CMD pip wheel --wheel-dir /tmp/wheels pip -$CHROOT_CMD pip wheel --wheel-dir /tmp/wheels -r /tmp/ipa-requirements.txt -$CHROOT_CMD pip wheel --no-index --pre --wheel-dir /tmp/wheels --find-links=/tmp/localpip --find-links=/tmp/wheels ironic-python-agent +$CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels setuptools +$CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels pip +$CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels -r /tmp/ipa-requirements.txt +$CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --no-index --pre --wheel-dir /tmp/wheels --find-links=/tmp/localpip --find-links=/tmp/wheels ironic-python-agent # Build tgt rm -rf $WORKDIR/build_files/tgt.tcz diff --git a/imagebuild/tinyipa/finalise-tinyipa.sh b/imagebuild/tinyipa/finalise-tinyipa.sh index 6de82232c..dca127df2 100755 --- a/imagebuild/tinyipa/finalise-tinyipa.sh +++ b/imagebuild/tinyipa/finalise-tinyipa.sh @@ -76,7 +76,9 @@ $CHROOT_CMD depmod -a `$WORKDIR/build_files/fakeuname -r` # If flag is set install the python now if $BUILD_AND_INSTALL_TINYIPA ; then - $CHROOT_CMD python /tmp/get-pip.py --no-wheel --no-index --find-links=file:///tmp/wheelhouse ironic_python_agent + cp -a $BUILDDIR/tmp/upper-constraints.txt $FINALDIR/tmp/upper-constraints.txt + $CHROOT_CMD python /tmp/get-pip.py -c /tmp/upper-constraints.txt --no-wheel --no-index --find-links=file:///tmp/wheelhouse ironic_python_agent + rm -rf $FINALDIR/tmp/upper-constraints.txt rm -rf $FINALDIR/tmp/wheelhouse rm -rf $FINALDIR/tmp/get-pip.py fi