From c66244ff555f9a364dc344073051861bbe74dc02 Mon Sep 17 00:00:00 2001 From: Julia Kreger Date: Tue, 16 Feb 2021 09:45:24 -0800 Subject: [PATCH] Remove PYOPTIMIZE_TINYIPA capability Oslo.Privsep requries pycparser, which does not support being loaded into python programs where the source is not available for doc string references. This is because one of the things that happens with an optomized python, is the files are "pre-compiled" and doc strings are removed. This is unfortunate, but just means the TinyIPA images are slightly larger, Testing shows this is approximately 5 MB. Change-Id: I3bdda3a8e595db8e84cd15bf083978c39514b1fa --- doc/source/admin/tinyipa.rst | 22 ------------- ...e-PYOPTIMIZE_TINYIPA-ebde63911d140209.yaml | 8 +++++ tinyipa/Makefile | 2 +- tinyipa/add-ssh-tinyipa.sh | 14 -------- tinyipa/build_files/bootlocal.sh | 2 -- tinyipa/finalise-tinyipa.sh | 32 ++----------------- 6 files changed, 11 insertions(+), 69 deletions(-) create mode 100644 releasenotes/notes/remove-PYOPTIMIZE_TINYIPA-ebde63911d140209.yaml diff --git a/doc/source/admin/tinyipa.rst b/doc/source/admin/tinyipa.rst index 520f9b0..7ce6d6b 100644 --- a/doc/source/admin/tinyipa.rst +++ b/doc/source/admin/tinyipa.rst @@ -100,28 +100,6 @@ or: Advanced options ---------------- -(De)Optimizing the image -~~~~~~~~~~~~~~~~~~~~~~~~ - -If you do not want the build script to preinstall everything into the ramdisk, -and instead load some things at runtime (this results in a slightly smaller -ramdisk), before running ``make`` or ``finalise-tinyipa.sh`` set: - -.. code-block:: bash - - export BUILD_AND_INSTALL_TINYIPA=false - -By default, building tinyIPA will compile most of the Python code to -optimized ``*.pyo`` files, completely remove most of ``*.py`` and ``*.pyc`` -files, and run ironic-python-agent with ``PYTHONOPTIMIZE=1`` -to save space on the ramdisk. If instead you want a normal Python experience -inside the image, for example for debugging/hacking on IPA in a running -ramdisk, before running ``make`` or ``build-tinyipa.sh`` set: - -.. code-block:: bash - - export PYOPTIMIZE_TINYIPA=false - Enabling/disabling SSH access to the ramdisk ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/releasenotes/notes/remove-PYOPTIMIZE_TINYIPA-ebde63911d140209.yaml b/releasenotes/notes/remove-PYOPTIMIZE_TINYIPA-ebde63911d140209.yaml new file mode 100644 index 0000000..dc4a814 --- /dev/null +++ b/releasenotes/notes/remove-PYOPTIMIZE_TINYIPA-ebde63911d140209.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Removes support for ``PYOPTIMIZE_TINYIPA``, which now casues the TinyIPA + image to be approximately 5 MB larger. This was due to an incompatability + with the ``oslo.privsep`` library dependency on ``pycparser`` which cannot + load into precompiled python programs where the original source embedded + documentation is not present. diff --git a/tinyipa/Makefile b/tinyipa/Makefile index ad0bf47..660e184 100644 --- a/tinyipa/Makefile +++ b/tinyipa/Makefile @@ -1,7 +1,7 @@ .PHONY: default all dependencies build finalise addssh iso clean clean_build clean_iso default: dependencies build finalise instance-images -all: dependencies build finalise iso instance-images +all: build finalise iso instance-images dependencies: ./install-deps.sh diff --git a/tinyipa/add-ssh-tinyipa.sh b/tinyipa/add-ssh-tinyipa.sh index 70c3864..1e8490e 100755 --- a/tinyipa/add-ssh-tinyipa.sh +++ b/tinyipa/add-ssh-tinyipa.sh @@ -104,15 +104,6 @@ function install_ssh { $TC_CHROOT_CMD chmod 600 /home/tc/.ssh/authorized_keys } -function fix_python_optimize { - if grep -q "PYTHONOPTIMIZE=1" "$REBUILDDIR/opt/bootlocal.sh"; then - # tinyipa was built with optimized Python environment, apply fixes - echo "PYTHONOPTIMIZE=1" | $TC_CHROOT_CMD tee -a /home/tc/.ssh/environment - echo "PermitUserEnvironment yes" | $CHROOT_CMD tee -a $SSHD_CONFIG_PATH - echo 'Defaults env_keep += "PYTHONOPTIMIZE"' | $CHROOT_CMD tee -a /etc/sudoers - fi -} - function rebuild_ramdisk { # Rebuild build directory into gz file @@ -134,11 +125,6 @@ setup_tce "$DST_DIR" # NOTE (pas-ha) default tinyipa is built without SSH access, enable it here install_ssh -# NOTE(pas-ha) default tinyipa is built with PYOPTIMIZE_TINYIPA=true and -# for Ansible+python to work we need to ensure that PYTHONOPTIMIZE=1 is -# set for all sessions from 'tc' user including those that are escalated -# with 'sudo' afterwards -fix_python_optimize cleanup_tce "$DST_DIR" rebuild_ramdisk diff --git a/tinyipa/build_files/bootlocal.sh b/tinyipa/build_files/bootlocal.sh index 49e7e3f..005d583 100755 --- a/tinyipa/build_files/bootlocal.sh +++ b/tinyipa/build_files/bootlocal.sh @@ -36,8 +36,6 @@ fi # Create ipa-rescue-config directory for rescue password sudo mkdir -p /etc/ipa-rescue-config -export PYTHONOPTIMIZE=1 - # Setup DHCP network configure_dhcp_network() { for pidfile in `ls /var/run/udhcpc*.pid`; do diff --git a/tinyipa/finalise-tinyipa.sh b/tinyipa/finalise-tinyipa.sh index 34dd482..6511459 100755 --- a/tinyipa/finalise-tinyipa.sh +++ b/tinyipa/finalise-tinyipa.sh @@ -13,7 +13,7 @@ INSTALL_SSH=${INSTALL_SSH:-true} AUTHORIZE_SSH=${AUTHORIZE_SSH:-false} SSH_PUBLIC_KEY=${SSH_PUBLIC_KEY:-} -PYOPTIMIZE_TINYIPA=${PYOPTIMIZE_TINYIPA:-true} +PYOPTIMIZE_TINYIPA=${PYOPTIMIZE_TINYIPA:-false} TINYIPA_REQUIRE_BIOSDEVNAME=${TINYIPA_REQUIRE_BIOSDEVNAME:-false} TINYIPA_REQUIRE_IPMITOOL=${TINYIPA_REQUIRE_IPMITOOL:-true} TINYIPA_UDEV_SETTLE_TIMEOUT=${TINYIPA_UDEV_SETTLE_TIMEOUT:-60} @@ -27,10 +27,6 @@ if [ -n "$PYTHON_EXTRA_SOURCES_DIR_LIST" ]; then IFS="," read -ra PKGDIRS <<< "$PYTHON_EXTRA_SOURCES_DIR_LIST" for PKGDIR in "${PKGDIRS[@]}"; do PKG=$(cd "$PKGDIR" ; python setup.py --name) - if [[ "$PKG" == "hardware" ]]; then - # hardware depends upon numpy which can't be optimised - PYOPTIMIZE_TINYIPA=false - fi done fi @@ -212,31 +208,7 @@ $CHROOT_CMD touch /var/lib/hwclock/adjtime $CHROOT_CMD chmod 640 /var/lib/hwclock/adjtime if $PYOPTIMIZE_TINYIPA; then - # Precompile all python - if [[ $USE_PYTHON3 == "True" ]]; then - set +e - $CHROOT_CMD /bin/bash -c "python3 -OO -m compileall /usr/local/lib/python3.6" - set -e - find $FINALDIR/usr/local/lib/python3.6 -name "*.py" -not -path "*ironic_python_agent/api/config.py" | sudo xargs --no-run-if-empty rm - find $FINALDIR/usr/local/lib/python3.6 -name "*.pyc" ! -name "*opt-2*" | sudo xargs --no-run-if-empty rm - sudo find $FINALDIR/usr/local/lib/python3.6 -type d -name __pycache__ -exec sh -c 'cd "$1"; for f in *; do mv -i "$f" .. ; done' find-sh {} \; - find $FINALDIR/usr/local/lib/python3.6 -name "*.cpython-36.opt-2*" | sed 'p;s/\.cpython-36\.opt-2//' | sudo xargs -n2 --no-run-if-empty mv - fi - set +e - $CHROOT_CMD /bin/bash -c "python -OO -m compileall /usr/local/lib/python2.7" - set -e - find $FINALDIR/usr/local/lib/python2.7 -name "*.py" -not -path "*ironic_python_agent/api/config.py" | sudo xargs --no-run-if-empty rm - find $FINALDIR/usr/local/lib/python2.7 -name "*.pyc" | sudo xargs --no-run-if-empty rm - if $INSTALL_SSH && $AUTHORIZE_SSH ; then - # NOTE(pas-ha) for Ansible+Python to work we need to ensure that - # PYTHONOPTIMIZE=1 is set for all sessions from 'tc' user including - # those that are elevated with 'sudo' afterwards - echo "PYTHONOPTIMIZE=1" | $TC_CHROOT_CMD tee -a /home/tc/.ssh/environment - echo "PermitUserEnvironment yes" | $CHROOT_CMD tee -a /usr/local/etc/ssh/sshd_config - echo 'Defaults env_keep += "PYTHONOPTIMIZE"' | $CHROOT_CMD tee -a /etc/sudoers - fi -else - sudo sed -i "s/PYTHONOPTIMIZE=1/PYTHONOPTIMIZE=0/" "$FINALDIR/opt/bootlocal.sh" + echo "WARNING: Precompilation is not compatible with oslo.privsep and is being ignored." fi # Delete unnecessary Babel .dat files