diff --git a/docker/python-builder/scripts/assemble b/docker/python-builder/scripts/assemble index fca4943e4f..683547a73f 100755 --- a/docker/python-builder/scripts/assemble +++ b/docker/python-builder/scripts/assemble @@ -48,29 +48,30 @@ function install_wheels { # that sdist which is exactly what we want. This triggers code # generation steps such as are found in zuul, since the sequencing # otherwise happens in a way that makes wheel content copying unhappy. - # `pip wheel` isn't used here because it puts all of the output - # in the output dir and not the wheel cache, so it's not - # possible to tell what is the wheel for the project and - # what is the wheel cache. - /tmp/venv/bin/python3 -m build -o /output/wheels ./ + # + # This is a separate step before we use pip wheel so that we can write + # the top level wheels to install into a different location than all of + # our dependencies. + /tmp/venv/bin/python3 -m build -o /output/toplevel_wheels ./ - # Install everything so that the wheel cache is populated with - # transitive depends. If a requirements.txt file exists, install - # it directly so that people can use git url syntax to do things - # like pick up patched but unreleased versions of dependencies. + # Pip wheel everything to build a collection of all wheels for all + # transitive depends. This ensures that the final install on the prod + # images can avoid installing any build depends. If a requirements.txt + # file exists, start there so that people can use git url syntax to do + # things like pick up patched but unreleased versions of dependencies. # Only do this for the main package (i.e. only write requirements # once). if [ -f /tmp/src/requirements.txt ] && [ ! -f /output/requirements.txt ] ; then - /tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels -r /tmp/src/requirements.txt + /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels -r /tmp/src/requirements.txt cp /tmp/src/requirements.txt /output/requirements.txt fi - /tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels /output/wheels/*whl + /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels /output/toplevel_wheels/*whl - # Install each of the extras so that we collect all possibly - # needed wheels in the wheel cache. get-extras-packages also + # Pip wheel each of the extras so that we collect all possibly + # needed wheels in the wheel output dir. get-extras-packages also # writes out the req files into /output/$extra/requirements.txt. for req in $(get-extras-packages) ; do - /tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels "$req" + /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels "$req" done } @@ -100,10 +101,10 @@ if [ -f /tmp/src/upper-constraints.txt ] ; then CONSTRAINTS="-c /tmp/src/upper-constraints.txt" fi -# If we got a list of packages, install them, otherwise install the -# main package. +# If we got a list of packages, get wheels for them, otherwise do +# this for the main package. if [[ $PACKAGES ]] ; then - /tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels $PACKAGES + /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels $PACKAGES for package in $PACKAGES ; do echo "$package" >> /output/packages.txt done @@ -115,7 +116,7 @@ else install_wheels fi -# go through ZUUL_SIBLINGS, if any, and build those wheels too +# Go through ZUUL_SIBLINGS, if any, and build those wheels too for sibling in ${ZUUL_SIBLINGS:-}; do pushd .zuul-siblings/${sibling} install_wheels diff --git a/docker/python-builder/scripts/install-from-bindep b/docker/python-builder/scripts/install-from-bindep index 7325b7860c..761b51f782 100755 --- a/docker/python-builder/scripts/install-from-bindep +++ b/docker/python-builder/scripts/install-from-bindep @@ -31,7 +31,10 @@ pip --version # to do things like pick up patched but unreleased versions # of dependencies. if [ -f /output/requirements.txt ] ; then - pip install $CONSTRAINTS --cache-dir=/output/wheels -r /output/requirements.txt + # --find-links will point pip at all of the wheels the assemble script + # downloaded or built. --no-index ensures that pip will only refer to + # the local set of wheels when looking for packages to install. + pip install $CONSTRAINTS --no-index --find-links=/output/wheels -r /output/requirements.txt fi # Add any requested extras to the list of things to install @@ -43,10 +46,10 @@ done if [ -f /output/packages.txt ] ; then # If a package list was passed to assemble, install that in the final # image. - pip install $CONSTRAINTS --cache-dir=/output/wheels -r /output/packages.txt $EXTRAS + pip install $CONSTRAINTS --no-index --find-links=/output/wheels -r /output/packages.txt $EXTRAS else - # Install the wheels. - pip install $CONSTRAINTS --cache-dir=/output/wheels /output/wheels/*.whl $EXTRAS + # Install the top level wheels. + pip install $CONSTRAINTS --no-index --find-links=/output/wheels /output/toplevel_wheels/*.whl $EXTRAS fi # clean up after ourselves