From 521df0eda02e72e32ceea930d80b45d89513956b Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Thu, 20 Oct 2022 07:50:57 -0700 Subject: [PATCH] Switch python-builder/python-base to pip wheel A recent change in pip wheel cache behavior had upstream pip indicating that we really should be using pip wheel instead. The reason we weren't using pip wheel appears to be that we wanted to infer what top level wheel to install via contents of a dir separate from our wheel output dir/wheel cache. Using pip wheel implies everything gets flattened into one location. We deal with this by having the build tool write all of the top level wheels we care about into a separate location. Later we can install all of the top level wheels while pointing find links at the larger set of deps in the dir created by pip wheel. Change-Id: Id9c674c1ec6fe5e72534549082e3adda9e286fd5 --- docker/python-builder/scripts/assemble | 37 ++++++++++--------- .../scripts/install-from-bindep | 11 ++++-- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/docker/python-builder/scripts/assemble b/docker/python-builder/scripts/assemble index 4d9aa0eb83..e10b9da035 100755 --- a/docker/python-builder/scripts/assemble +++ b/docker/python-builder/scripts/assemble @@ -48,29 +48,30 @@ function install_wheels { # that sdist which is exactly what we want. This triggers code # generation steps such as are found in zuul, since the sequencing # otherwise happens in a way that makes wheel content copying unhappy. - # `pip wheel` isn't used here because it puts all of the output - # in the output dir and not the wheel cache, so it's not - # possible to tell what is the wheel for the project and - # what is the wheel cache. - /tmp/venv/bin/python3 -m build -o /output/wheels ./ + # + # This is a separate step before we use pip wheel so that we can write + # the top level wheels to install into a different location than all of + # our dependencies. + /tmp/venv/bin/python3 -m build -o /output/toplevel_wheels ./ - # Install everything so that the wheel cache is populated with - # transitive depends. If a requirements.txt file exists, install - # it directly so that people can use git url syntax to do things - # like pick up patched but unreleased versions of dependencies. + # Pip wheel everything to build a collection of all wheels for all + # transitive depends. This ensures that the final install on the prod + # images can avoid installing any build depends. If a requirements.txt + # file exists, start there so that people can use git url syntax to do + # things like pick up patched but unreleased versions of dependencies. # Only do this for the main package (i.e. only write requirements # once). if [ -f /tmp/src/requirements.txt ] && [ ! -f /output/requirements.txt ] ; then - /tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels -r /tmp/src/requirements.txt + /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels -r /tmp/src/requirements.txt cp /tmp/src/requirements.txt /output/requirements.txt fi - /tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels /output/wheels/*whl + /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels /output/toplevel_wheels/*whl - # Install each of the extras so that we collect all possibly - # needed wheels in the wheel cache. get-extras-packages also + # Pip wheel each of the extras so that we collect all possibly + # needed wheels in the wheel output dir. get-extras-packages also # writes out the req files into /output/$extra/requirements.txt. for req in $(get-extras-packages) ; do - /tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels "$req" + /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels "$req" done } @@ -98,10 +99,10 @@ if [ -f /tmp/src/upper-constraints.txt ] ; then CONSTRAINTS="-c /tmp/src/upper-constraints.txt" fi -# If we got a list of packages, install them, otherwise install the -# main package. +# If we got a list of packages, get wheels for them, otherwise do +# this for the main package. if [[ $PACKAGES ]] ; then - /tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels $PACKAGES + /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels $PACKAGES for package in $PACKAGES ; do echo "$package" >> /output/packages.txt done @@ -113,7 +114,7 @@ else install_wheels fi -# go through ZUUL_SIBLINGS, if any, and build those wheels too +# Go through ZUUL_SIBLINGS, if any, and build those wheels too for sibling in ${ZUUL_SIBLINGS:-}; do pushd .zuul-siblings/${sibling} install_wheels diff --git a/docker/python-builder/scripts/install-from-bindep b/docker/python-builder/scripts/install-from-bindep index 916d0735ae..7214e640f5 100755 --- a/docker/python-builder/scripts/install-from-bindep +++ b/docker/python-builder/scripts/install-from-bindep @@ -29,7 +29,10 @@ fi # to do things like pick up patched but unreleased versions # of dependencies. if [ -f /output/requirements.txt ] ; then - pip install $CONSTRAINTS --cache-dir=/output/wheels -r /output/requirements.txt + # --find-links will point pip at all of the wheels the assemble script + # downloaded or built. --no-index ensures that pip will only refer to + # the local set of wheels when looking for packages to install. + pip install $CONSTRAINTS --no-index --find-links=/output/wheels -r /output/requirements.txt fi # Add any requested extras to the list of things to install @@ -41,10 +44,10 @@ done if [ -f /output/packages.txt ] ; then # If a package list was passed to assemble, install that in the final # image. - pip install $CONSTRAINTS --cache-dir=/output/wheels -r /output/packages.txt $EXTRAS + pip install $CONSTRAINTS --no-index --find-links=/output/wheels -r /output/packages.txt $EXTRAS else - # Install the wheels. - pip install $CONSTRAINTS --cache-dir=/output/wheels /output/wheels/*.whl $EXTRAS + # Install the top level wheels. + pip install $CONSTRAINTS --no-index --find-links=/output/wheels /output/toplevel_wheels/*.whl $EXTRAS fi # clean up after ourselves