From c51b43806a497449681bbe151d7c02f266a5e070 Mon Sep 17 00:00:00 2001
From: Michael Davies <michael@the-davies.net>
Date: Tue, 3 Sep 2019 19:11:41 +0930
Subject: [PATCH] Allow extra python pakgs into tinyipa

Allow the user to specify additional python packages via the
PYTHON_EXTRA_SOURCES_DIR_LIST env variable.  The list of packages
needs to be a comma separated list of directories of local checkouts.

This allows additional optional packages, such as 'hardware', to be added
into a tinyipa build, even though they aren't specified in a
requirements.txt file elsewhere.

Also turn off python package optimisation if 'hardware' is included, due
to one of its dependencies.

Change-Id: I001aa9c116bfd78afd70a489a69c53c60f6a265b
---
 tinyipa/build-tinyipa.sh    | 29 +++++++++++++++++++++++++++++
 tinyipa/finalise-tinyipa.sh | 21 +++++++++++++++++++++
 2 files changed, 50 insertions(+)

diff --git a/tinyipa/build-tinyipa.sh b/tinyipa/build-tinyipa.sh
index b011bca..c772634 100755
--- a/tinyipa/build-tinyipa.sh
+++ b/tinyipa/build-tinyipa.sh
@@ -11,6 +11,9 @@ TINYIPA_REQUIRE_IPMITOOL=${TINYIPA_REQUIRE_IPMITOOL:-true}
 IRONIC_LIB_SOURCE=${IRONIC_LIB_SOURCE:-}
 USE_PYTHON3=${USE_PYTHON3:-True}
 
+# PYTHON_EXTRA_SOURCES_DIR_LIST is a csv list of python package dirs to include
+PYTHON_EXTRA_SOURCES_DIR_LIST=${PYTHON_EXTRA_SOURCES_DIR_LIST:-}
+
 CHROOT_PATH="/tmp/overides:/usr/local/sbin:/usr/local/bin:/apps/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 CHROOT_CMD="sudo chroot $BUILDDIR /usr/bin/env -i PATH=$CHROOT_PATH http_proxy=$http_proxy https_proxy=$https_proxy no_proxy=$no_proxy"
 
@@ -89,6 +92,20 @@ if [ -n "$IRONIC_LIB_SOURCE" ]; then
     popd
 fi
 
+if [ -n "$PYTHON_EXTRA_SOURCES_DIR_LIST" ]; then
+    IFS="," read -ra PKGDIRS <<< "$PYTHON_EXTRA_SOURCES_DIR_LIST"
+    for PKGDIR in "${PKGDIRS[@]}"; do
+        PKG=$(cd "$PKGDIR" ; python setup.py --name)
+        pushd "$PKGDIR"
+        rm -rf *.egg-info
+        python setup.py sdist --dist-dir "$BUILDDIR/tmp/localpip" --quiet
+        if [[ -r requirements.txt ]]; then
+            cp requirements.txt $BUILDDIR/tmp/${PKG}-requirements.txt
+        fi
+        popd
+    done
+fi
+
 imagebuild/common/generate_upper_constraints.sh upper-constraints.txt
 if [ -n "$IRONIC_LIB_SOURCE" ]; then
     sed -i '/ironic-lib/d' upper-constraints.txt $BUILDDIR/tmp/ipa-requirements.txt
@@ -156,6 +173,18 @@ if [ -n "$IRONIC_LIB_SOURCE" ]; then
     $CHROOT_CMD ${PIP_COMMAND} wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels -r /tmp/ironic-lib-requirements.txt
     $CHROOT_CMD ${PIP_COMMAND} wheel -c /tmp/upper-constraints.txt --no-index --pre --wheel-dir /tmp/wheels --find-links=/tmp/localpip --find-links=/tmp/wheels ironic-lib
 fi
+
+if [ -n "$PYTHON_EXTRA_SOURCES_DIR_LIST" ]; then
+    IFS="," read -ra PKGDIRS <<< "$PYTHON_EXTRA_SOURCES_DIR_LIST"
+    for PKGDIR in "${PKGDIRS[@]}"; do
+        PKG=$(cd "$PKGDIR" ; python setup.py --name)
+        if [[ -r $BUILDDIR/tmp/${PKG}-requirements.txt ]]; then
+            $CHROOT_CMD ${PIP_COMMAND} wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels -r /tmp/${PKG}-requirements.txt
+        fi
+        $CHROOT_CMD ${PIP_COMMAND} wheel -c /tmp/upper-constraints.txt --no-index --pre --wheel-dir /tmp/wheels --find-links=/tmp/localpip --find-links=/tmp/wheels ${PKG}
+    done
+fi
+
 $CHROOT_CMD ${PIP_COMMAND} wheel -c /tmp/upper-constraints.txt --no-index --pre --wheel-dir /tmp/wheels --find-links=/tmp/localpip --find-links=/tmp/wheels ironic-python-agent
 echo Resulting wheels:
 ls -1 $BUILDDIR/tmp/wheels
diff --git a/tinyipa/finalise-tinyipa.sh b/tinyipa/finalise-tinyipa.sh
index dace708..4b71f45 100755
--- a/tinyipa/finalise-tinyipa.sh
+++ b/tinyipa/finalise-tinyipa.sh
@@ -19,9 +19,21 @@ TINYIPA_REQUIRE_IPMITOOL=${TINYIPA_REQUIRE_IPMITOOL:-true}
 TINYIPA_UDEV_SETTLE_TIMEOUT=${TINYIPA_UDEV_SETTLE_TIMEOUT:-20}
 USE_PYTHON3=${USE_PYTHON3:-True}
 
+PYTHON_EXTRA_SOURCES_DIR_LIST=${PYTHON_EXTRA_SOURCES_DIR_LIST:-}
 
 echo "Finalising tinyipa:"
 
+if [ -n "$PYTHON_EXTRA_SOURCES_DIR_LIST" ]; then
+    IFS="," read -ra PKGDIRS <<< "$PYTHON_EXTRA_SOURCES_DIR_LIST"
+    for PKGDIR in "${PKGDIRS[@]}"; do
+        PKG=$(cd "$PKGDIR" ; python setup.py --name)
+        if [[ "$PKG" == "hardware" ]]; then
+            # hardware depends upon numpy which can't be optimised
+            PYOPTIMIZE_TINYIPA=false
+        fi
+    done
+fi
+
 if $AUTHORIZE_SSH ; then
     echo "Validating location of public SSH key"
     if [ -n "$SSH_PUBLIC_KEY" ]; then
@@ -140,7 +152,16 @@ $CHROOT_CMD ${TINYIPA_PYTHON_EXE} -m ensurepip --upgrade
 
 # If flag is set install python now
 if $BUILD_AND_INSTALL_TINYIPA ; then
+    if [ -n "$PYTHON_EXTRA_SOURCES_DIR_LIST" ]; then
+        IFS="," read -ra PKGDIRS <<< "$PYTHON_EXTRA_SOURCES_DIR_LIST"
+        for PKGDIR in "${PKGDIRS[@]}"; do
+            PKG=$(cd "$PKGDIR" ; python setup.py --name)
+            $CHROOT_CMD $PIP_COMMAND install --no-index --find-links=file:///tmp/wheelhouse --pre $PKG
+        done
+    fi
+
     $CHROOT_CMD $PIP_COMMAND install --no-index --find-links=file:///tmp/wheelhouse --pre ironic_python_agent
+
     rm -rf $FINALDIR/tmp/wheelhouse
 fi