Build tinyipa with python3

This patch uses ``USE_PYTHON3`` variable to build
Ironic Python Agent with python 3 inside a tinycore
image.
It modifies the content of ``buildreqs.lst`` and
``finalreqs.lst``, adding specific package files
based on the python version used to build IPA.
Since we're using python 3.6, bytecode optimized
files have .pyc extension and are located under a
__pycache__ dir [1]; as an optimization option,to
reduce the size of the final image, all bytecode
generated files will be moved to the original
source location and the source code will be
removed.

[1] https://www.python.org/dev/peps/pep-0488/

Change-Id: I3448a5f41eff6736274308f924d9ad8f059e6c44
Story: #2002598
Task: #22210
Depends-On: https://review.openstack.org/634646

imagebuild/tinyipa/build-tinyipa.sh

@@ -9,6 +9,7 @@ TINYCORE_MIRROR_URL=${TINYCORE_MIRROR_URL:-}
 TINYIPA_REQUIRE_BIOSDEVNAME=${TINYIPA_REQUIRE_BIOSDEVNAME:-false}
 TINYIPA_REQUIRE_IPMITOOL=${TINYIPA_REQUIRE_IPMITOOL:-true}
 IRONIC_LIB_SOURCE=${IRONIC_LIB_SOURCE:-}
+USE_PYTHON3=${USE_PYTHON3:-False}
 
 CHROOT_PATH="/tmp/overides:/usr/local/sbin:/usr/local/bin:/apps/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 CHROOT_CMD="sudo chroot $BUILDDIR /usr/bin/env -i PATH=$CHROOT_PATH http_proxy=$http_proxy https_proxy=$https_proxy no_proxy=$no_proxy"
@@ -74,7 +75,9 @@ mkdir -p "$BUILDDIR/tmp/localpip"
 # Download IPA and requirements
 cd ../..
 rm -rf *.egg-info
+pwd
 python setup.py sdist --dist-dir "$BUILDDIR/tmp/localpip" --quiet
+ls $BUILDDIR/tmp/localpip || true
 cp requirements.txt $BUILDDIR/tmp/ipa-requirements.txt
 
 if [ -n "$IRONIC_LIB_SOURCE" ]; then
@@ -127,20 +130,32 @@ $CHROOT_CMD chmod a+rwx /etc/sysconfig/tcuser
 mkdir $BUILDDIR/tmp/overides
 cp $WORKDIR/build_files/fakeuname $BUILDDIR/tmp/overides/uname
 
+PY_REQS="buildreqs_python2.lst"
+if [[ $USE_PYTHON3 == "True" ]]; then
+    PY_REQS="buildreqs_python3.lst"
+fi
+
 while read line; do
     sudo chroot --userspec=$TC:$STAFF $BUILDDIR /usr/bin/env -i PATH=$CHROOT_PATH http_proxy=$http_proxy https_proxy=$https_proxy no_proxy=$no_proxy tce-load -wci $line
-done < $WORKDIR/build_files/buildreqs.lst
+done < <(paste $WORKDIR/build_files/$PY_REQS $WORKDIR/build_files/buildreqs.lst)
+
+PIP_COMMAND="pip"
+TINYIPA_PYTHON_EXE="python"
+if [[ $USE_PYTHON3 == "True" ]]; then
+    PIP_COMMAND="pip3"
+    TINYIPA_PYTHON_EXE="python3"
+fi
 
 # Build python wheels
-$CHROOT_CMD python -m ensurepip
+$CHROOT_CMD ${TINYIPA_PYTHON_EXE} -m ensurepip
-$CHROOT_CMD pip install --upgrade pip wheel
+$CHROOT_CMD ${PIP_COMMAND} install --upgrade pip wheel
-$CHROOT_CMD pip install pbr
+$CHROOT_CMD ${PIP_COMMAND} install pbr
-$CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels -r /tmp/ipa-requirements.txt
+$CHROOT_CMD ${PIP_COMMAND} wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels -r /tmp/ipa-requirements.txt
 if [ -n "$IRONIC_LIB_SOURCE" ]; then
-    $CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels -r /tmp/ironic-lib-requirements.txt
+    $CHROOT_CMD ${PIP_COMMAND} wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels -r /tmp/ironic-lib-requirements.txt
-    $CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --no-index --pre --wheel-dir /tmp/wheels --find-links=/tmp/localpip --find-links=/tmp/wheels ironic-lib
+    $CHROOT_CMD ${PIP_COMMAND} wheel -c /tmp/upper-constraints.txt --no-index --pre --wheel-dir /tmp/wheels --find-links=/tmp/localpip --find-links=/tmp/wheels ironic-lib
 fi
-$CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --no-index --pre --wheel-dir /tmp/wheels --find-links=/tmp/localpip --find-links=/tmp/wheels ironic-python-agent
+$CHROOT_CMD ${PIP_COMMAND} wheel -c /tmp/upper-constraints.txt --no-index --pre --wheel-dir /tmp/wheels --find-links=/tmp/localpip --find-links=/tmp/wheels ironic-python-agent
 echo Resulting wheels:
 ls -1 $BUILDDIR/tmp/wheels
 

imagebuild/tinyipa/build_files/bootlocal.sh

@@ -20,7 +20,11 @@ fi
 
 # Install IPA and dependecies
 if ! type "ironic-python-agent" > /dev/null ; then
-    pip install --no-index --find-links=file:///tmp/wheelhouse ironic_python_agent
+    PIP_COMMAND="pip"
+    if hash pip3 2>/dev/null; then
+        PIP_COMMAND="pip3"
+    fi
+    $PIP_COMMAND install --no-index --find-links=file:///tmp/wheelhouse ironic_python_agent
 fi
 
 # Create ipa-rescue-config directory for rescue password

imagebuild/tinyipa/build_files/buildreqs.lst

@@ -18,8 +18,6 @@ parted.tcz
 pciutils.tcz
 pixman-dev.tcz
 pkg-config.tcz
-python-dev.tcz
-python.tcz
 raid-dm-4.8.17-tinycore64.tcz
 scsi-4.8.17-tinycore64.tcz
 udev-lib.tcz

imagebuild/tinyipa/build_files/buildreqs_python2.lst

@@ -0,0 +1,2 @@
+python.tcz
+python-dev.tcz

imagebuild/tinyipa/build_files/buildreqs_python3.lst

@@ -0,0 +1,2 @@
+python3.6.tcz
+python3.6-dev.tcz

imagebuild/tinyipa/build_files/finalreqs.lst

@@ -6,7 +6,6 @@ hdparm.tcz
 iproute2.tcz
 parted.tcz
 popt.tcz
-python.tcz
 pciutils.tcz
 raid-dm-4.8.17-tinycore64.tcz
 scsi-4.8.17-tinycore64.tcz

imagebuild/tinyipa/build_files/finalreqs_python2.lst

@@ -0,0 +1 @@
+python.tcz

imagebuild/tinyipa/build_files/finalreqs_python3.lst

@@ -0,0 +1,2 @@
+python3.6.tcz
+expat2.tcz

imagebuild/tinyipa/finalise-tinyipa.sh

@@ -24,6 +24,8 @@ PYOPTIMIZE_TINYIPA=${PYOPTIMIZE_TINYIPA:-true}
 TINYIPA_REQUIRE_BIOSDEVNAME=${TINYIPA_REQUIRE_BIOSDEVNAME:-false}
 TINYIPA_REQUIRE_IPMITOOL=${TINYIPA_REQUIRE_IPMITOOL:-true}
 TINYIPA_UDEV_SETTLE_TIMEOUT=${TINYIPA_UDEV_SETTLE_TIMEOUT:-20}
+USE_PYTHON3=${USE_PYTHON3:-False}
+
 
 echo "Finalising tinyipa:"
 
@@ -85,9 +87,14 @@ fi
 mkdir $FINALDIR/tmp/overides
 cp $WORKDIR/build_files/fakeuname $FINALDIR/tmp/overides/uname
 
+PY_REQS="finalreqs_python2.lst"
+if [[ $USE_PYTHON3 == "True" ]]; then
+    PY_REQS="finalreqs_python3.lst"
+fi
+
 while read line; do
     $TC_CHROOT_CMD tce-load -wic $line
-done < $WORKDIR/build_files/finalreqs.lst
+done < <(paste $WORKDIR/build_files/finalreqs.lst $WORKDIR/build_files/$PY_REQS)
 
 if $INSTALL_SSH ; then
     # Install and configure bare minimum for SSH access
@@ -128,12 +135,19 @@ fi
 # Ensure tinyipa picks up installed kernel modules
 $CHROOT_CMD depmod -a `$WORKDIR/build_files/fakeuname -r`
 
-# Install pip
+PIP_COMMAND="pip"
-$CHROOT_CMD python -m ensurepip --upgrade
+TINYIPA_PYTHON_EXE="python"
+if [[ $USE_PYTHON3 == "True" ]]; then
+    PIP_COMMAND="pip3"
+    TINYIPA_PYTHON_EXE="python3"
+fi
 
-# If flag is set install the python now
+# Install pip
+$CHROOT_CMD ${TINYIPA_PYTHON_EXE} -m ensurepip --upgrade
+
+# If flag is set install python now
 if $BUILD_AND_INSTALL_TINYIPA ; then
-    $CHROOT_CMD pip install --no-index --find-links=file:///tmp/wheelhouse --pre ironic_python_agent
+    $CHROOT_CMD $PIP_COMMAND install --no-index --find-links=file:///tmp/wheelhouse --pre ironic_python_agent
     rm -rf $FINALDIR/tmp/wheelhouse
 fi
 
@@ -156,6 +170,15 @@ sudo sed -i '/# Main/a NOZSWAP=1' "$FINALDIR/etc/init.d/tc-config"
 
 if $PYOPTIMIZE_TINYIPA; then
     # Precompile all python
+    if [[ $USE_PYTHON3 == "True" ]]; then
+        set +e
+        $CHROOT_CMD /bin/bash -c "python3 -OO -m compileall /usr/local/lib/python3.6"
+        set -e
+        find $FINALDIR/usr/local/lib/python3.6 -name "*.py" -not -path "*ironic_python_agent/api/config.py" | sudo xargs --no-run-if-empty rm
+        find $FINALDIR/usr/local/lib/python3.6 -name "*.pyc" ! -name "*opt-2*" | sudo xargs --no-run-if-empty rm
+        sudo find $FINALDIR/usr/local/lib/python3.6 -type d -name __pycache__ -exec sh -c 'cd "$1"; for f in *; do mv -i "$f" .. ; done' find-sh {} \;
+        find $FINALDIR/usr/local/lib/python3.6 -name "*.cpython-36.opt-2*" | sed 'p;s/\.cpython-36\.opt-2//' | sudo xargs -n2 --no-run-if-empty mv
+    fi
     set +e
     $CHROOT_CMD /bin/bash -c "python -OO -m compileall /usr/local/lib/python2.7"
     set -e

releasenotes/notes/build-tinyipa-with-python3-d4a64aa18f970968.yaml

@@ -0,0 +1,4 @@
+---
+other:
+  - Updates the tinyipa build scripts to support building and running Ironic
+    Python Agent with python3.6.

zuul.d/ironic-jobs.yaml

@@ -54,6 +54,7 @@
     parent: ironic-ipa-base
     timeout: 7200
     vars:
+      devstack_localrc:
         IRONIC_DEPLOY_DRIVER: ipmi
         IRONIC_DEFAULT_DEPLOY_INTERFACE: iscsi