Merge "Switch python-builder/python-base to pip wheel"

docker/python-builder/scripts/assemble

@@ -48,29 +48,30 @@ function install_wheels {
     # that sdist which is exactly what we want. This triggers code
     # generation steps such as are found in zuul, since the sequencing
     # otherwise happens in a way that makes wheel content copying unhappy.
-    # `pip wheel` isn't used here because it puts all of the output
-    # in the output dir and not the wheel cache, so it's not
-    # possible to tell what is the wheel for the project and
-    # what is the wheel cache.
-    /tmp/venv/bin/python3 -m build -o /output/wheels ./
+    #
+    # This is a separate step before we use pip wheel so that we can write
+    # the top level wheels to install into a different location than all of
+    # our dependencies.
+    /tmp/venv/bin/python3 -m build -o /output/toplevel_wheels ./
 
-    # Install everything so that the wheel cache is populated with
-    # transitive depends. If a requirements.txt file exists, install
-    # it directly so that people can use git url syntax to do things
-    # like pick up patched but unreleased versions of dependencies.
+    # Pip wheel everything to build a collection of all wheels for all
+    # transitive depends. This ensures that the final install on the prod
+    # images can avoid installing any build depends. If a requirements.txt
+    # file exists, start there so that people can use git url syntax to do
+    # things like pick up patched but unreleased versions of dependencies.
     # Only do this for the main package (i.e. only write requirements
     # once).
     if [ -f /tmp/src/requirements.txt ] && [ ! -f /output/requirements.txt ] ; then
-        /tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels -r /tmp/src/requirements.txt
+        /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels -r /tmp/src/requirements.txt
         cp /tmp/src/requirements.txt /output/requirements.txt
     fi
-    /tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels /output/wheels/*whl
+    /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels /output/toplevel_wheels/*whl
 
-    # Install each of the extras so that we collect all possibly
-    # needed wheels in the wheel cache. get-extras-packages also
+    # Pip wheel each of the extras so that we collect all possibly
+    # needed wheels in the wheel output dir. get-extras-packages also
     # writes out the req files into /output/$extra/requirements.txt.
     for req in $(get-extras-packages) ; do
-        /tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels "$req"
+        /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels "$req"
     done
 }
 
@@ -100,10 +101,10 @@ if [ -f /tmp/src/upper-constraints.txt ] ; then
     CONSTRAINTS="-c /tmp/src/upper-constraints.txt"
 fi
 
-# If we got a list of packages, install them, otherwise install the
-# main package.
+# If we got a list of packages, get wheels for them, otherwise do
+# this for the main package.
 if [[ $PACKAGES ]] ; then
-    /tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels $PACKAGES
+    /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels $PACKAGES
     for package in $PACKAGES ; do
       echo "$package" >> /output/packages.txt
     done
@@ -115,7 +116,7 @@ else
     install_wheels
 fi
 
-# go through ZUUL_SIBLINGS, if any, and build those wheels too
+# Go through ZUUL_SIBLINGS, if any, and build those wheels too
 for sibling in ${ZUUL_SIBLINGS:-}; do
     pushd .zuul-siblings/${sibling}
     install_wheels

docker/python-builder/scripts/install-from-bindep

@@ -31,7 +31,10 @@ pip --version
 # to do things like pick up patched but unreleased versions
 # of dependencies.
 if [ -f /output/requirements.txt ] ; then
-    pip install $CONSTRAINTS --cache-dir=/output/wheels -r /output/requirements.txt
+    # --find-links will point pip at all of the wheels the assemble script
+    # downloaded or built. --no-index ensures that pip will only refer to
+    # the local set of wheels when looking for packages to install.
+    pip install $CONSTRAINTS --no-index --find-links=/output/wheels -r /output/requirements.txt
 fi
 
 # Add any requested extras to the list of things to install
@@ -43,10 +46,10 @@ done
 if [ -f /output/packages.txt ] ; then
   # If a package list was passed to assemble, install that in the final
   # image.
-  pip install $CONSTRAINTS --cache-dir=/output/wheels -r /output/packages.txt $EXTRAS
+  pip install $CONSTRAINTS --no-index --find-links=/output/wheels -r /output/packages.txt $EXTRAS
 else
-  # Install the wheels.
-  pip install $CONSTRAINTS --cache-dir=/output/wheels /output/wheels/*.whl $EXTRAS
+  # Install the top level wheels.
+  pip install $CONSTRAINTS --no-index --find-links=/output/wheels /output/toplevel_wheels/*.whl $EXTRAS
 fi
 
 # clean up after ourselves