opendev/system-config
Code Issues Proposed changes

Merge "Switch python-builder/python-base to pip wheel"

Browse Source
This commit is contained in:
Zuul 2022-11-21 18:35:21 +00:00 committed by Gerrit Code Review
commit c89ffc3d6f
2 changed files with 26 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
docker/python-builder/scripts/assemble
View File

@ -48,29 +48,30 @@ function install_wheels {
# that sdist which is exactly what we want. This triggers code # that sdist which is exactly what we want. This triggers code
# generation steps such as are found in zuul, since the sequencing # generation steps such as are found in zuul, since the sequencing
# otherwise happens in a way that makes wheel content copying unhappy. # otherwise happens in a way that makes wheel content copying unhappy.
# `pip wheel` isn't used here because it puts all of the output #
# in the output dir and not the wheel cache, so it's not # This is a separate step before we use pip wheel so that we can write
# possible to tell what is the wheel for the project and # the top level wheels to install into a different location than all of
# what is the wheel cache. # our dependencies.
/tmp/venv/bin/python3 -m build -o /output/wheels ./ /tmp/venv/bin/python3 -m build -o /output/toplevel_wheels ./
# Install everything so that the wheel cache is populated with # Pip wheel everything to build a collection of all wheels for all
# transitive depends. If a requirements.txt file exists, install # transitive depends. This ensures that the final install on the prod
# it directly so that people can use git url syntax to do things # images can avoid installing any build depends. If a requirements.txt
# like pick up patched but unreleased versions of dependencies. # file exists, start there so that people can use git url syntax to do
# things like pick up patched but unreleased versions of dependencies.
# Only do this for the main package (i.e. only write requirements # Only do this for the main package (i.e. only write requirements
# once). # once).
if [ -f /tmp/src/requirements.txt ] && [ ! -f /output/requirements.txt ] ; then if [ -f /tmp/src/requirements.txt ] && [ ! -f /output/requirements.txt ] ; then
/tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels -r /tmp/src/requirements.txt /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels -r /tmp/src/requirements.txt
cp /tmp/src/requirements.txt /output/requirements.txt cp /tmp/src/requirements.txt /output/requirements.txt
fi fi
/tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels /output/wheels/*whl /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels /output/toplevel_wheels/*whl
# Install each of the extras so that we collect all possibly # Pip wheel each of the extras so that we collect all possibly
# needed wheels in the wheel cache. get-extras-packages also # needed wheels in the wheel output dir. get-extras-packages also
# writes out the req files into /output/$extra/requirements.txt. # writes out the req files into /output/$extra/requirements.txt.
for req in $(get-extras-packages) ; do for req in $(get-extras-packages) ; do
/tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels "$req" /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels "$req"
done done
} }
@ -100,10 +101,10 @@ if [ -f /tmp/src/upper-constraints.txt ] ; then
CONSTRAINTS="-c /tmp/src/upper-constraints.txt" CONSTRAINTS="-c /tmp/src/upper-constraints.txt"
fi fi
# If we got a list of packages, install them, otherwise install the # If we got a list of packages, get wheels for them, otherwise do
# main package. # this for the main package.
if [[ $PACKAGES ]] ; then if [[ $PACKAGES ]] ; then
/tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels $PACKAGES /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels $PACKAGES
for package in $PACKAGES ; do for package in $PACKAGES ; do
echo "$package" >> /output/packages.txt echo "$package" >> /output/packages.txt
done done
@ -115,7 +116,7 @@ else
install_wheels install_wheels
fi fi
# go through ZUUL_SIBLINGS, if any, and build those wheels too # Go through ZUUL_SIBLINGS, if any, and build those wheels too
for sibling in ${ZUUL_SIBLINGS:-}; do for sibling in ${ZUUL_SIBLINGS:-}; do
pushd .zuul-siblings/${sibling} pushd .zuul-siblings/${sibling}
install_wheels install_wheels

11
docker/python-builder/scripts/install-from-bindep
View File

@ -31,7 +31,10 @@ pip --version
# to do things like pick up patched but unreleased versions # to do things like pick up patched but unreleased versions
# of dependencies. # of dependencies.
if [ -f /output/requirements.txt ] ; then if [ -f /output/requirements.txt ] ; then
pip install $CONSTRAINTS --cache-dir=/output/wheels -r /output/requirements.txt # --find-links will point pip at all of the wheels the assemble script
# downloaded or built. --no-index ensures that pip will only refer to
# the local set of wheels when looking for packages to install.
pip install $CONSTRAINTS --no-index --find-links=/output/wheels -r /output/requirements.txt
fi fi
# Add any requested extras to the list of things to install # Add any requested extras to the list of things to install
@ -43,10 +46,10 @@ done
if [ -f /output/packages.txt ] ; then if [ -f /output/packages.txt ] ; then
# If a package list was passed to assemble, install that in the final # If a package list was passed to assemble, install that in the final
# image. # image.
pip install $CONSTRAINTS --cache-dir=/output/wheels -r /output/packages.txt $EXTRAS pip install $CONSTRAINTS --no-index --find-links=/output/wheels -r /output/packages.txt $EXTRAS
else else
# Install the wheels. # Install the top level wheels.
pip install $CONSTRAINTS --cache-dir=/output/wheels /output/wheels/*.whl $EXTRAS pip install $CONSTRAINTS --no-index --find-links=/output/wheels /output/toplevel_wheels/*.whl $EXTRAS
fi fi
# clean up after ourselves # clean up after ourselves