Add option to install everything in global venvs

Since we are python3 only for openstack we create a single python3
virtualenv to install all the packages into. This gives us the benefits
of installing into a virtualenv while still ensuring coinstallability.
This is a major change and will likely break many things.

There are several reasons for this. The change that started this effort
was pip stopped uninstalling packages which used distutils to generate
their package installation. Many distro packages do this which meant
that pip installed packages and distro packages could not coexist in the
global install space. More recently git has made pip installing repos as
root more difficult due to file ownership concerns.

Currently the switch to the global venv is optional, but if we go down
this path we should very quickly remove the old global installation
method as it has only caused us problems.

Major hurdles we have to get over are convincing rootwrap to trust
binaries in the virtualenvs (so you'll notice we update rootwrap
configs).

Some distros still have issues, keep them using the old setup for now.

Depends-On: https://review.opendev.org/c/openstack/grenade/+/880266
Co-Authored-By: Dr. Jens Harbott <frickler@offenerstapel.de>
Change-Id: If9bc7ba45522189d03f19b86cb681bb150ee2f25

.zuul.yaml

@@ -700,6 +700,9 @@
     # TODO(kopecmartin) n-v until the following is resolved:
     # https://bugs.launchpad.net/neutron/+bug/1979047
     voting: false
+    vars:
+      devstack_localrc:
+        GLOBAL_VENV: false
 
 - job:
     name: devstack-platform-debian-bullseye
@@ -709,6 +712,9 @@
     timeout: 9000
     vars:
       configure_swap_size: 4096
+      devstack_localrc:
+        # TODO(frickler): drop this once wheel build is fixed
+        MYSQL_GATHER_PERFORMANCE: false
 
 - job:
     name: devstack-platform-rocky-blue-onyx
@@ -718,6 +724,8 @@
     timeout: 9000
     vars:
       configure_swap_size: 4096
+      devstack_localrc:
+        GLOBAL_VENV: false
 
 - job:
     name: devstack-platform-ubuntu-focal

files/apache-horizon.template

@@ -39,4 +39,5 @@
     CustomLog /var/log/%APACHE_NAME%/horizon_access.log combined
 </VirtualHost>
 
+%WSGIPYTHONHOME%
 WSGISocketPrefix /var/run/%APACHE_NAME%

functions-common

@@ -1522,6 +1522,7 @@ function write_user_unit_file {
     mkdir -p $SYSTEMD_DIR
 
     iniset -sudo $unitfile "Unit" "Description" "Devstack $service"
+    iniset -sudo $unitfile "Service" "Environment" "\"PATH=$PATH\""
     iniset -sudo $unitfile "Service" "User" "$user"
     iniset -sudo $unitfile "Service" "ExecStart" "$command"
     iniset -sudo $unitfile "Service" "KillMode" "process"
@@ -1549,6 +1550,7 @@ function write_uwsgi_user_unit_file {
     mkdir -p $SYSTEMD_DIR
 
     iniset -sudo $unitfile "Unit" "Description" "Devstack $service"
+    iniset -sudo $unitfile "Service" "Environment" "\"PATH=$PATH\""
     iniset -sudo $unitfile "Service" "SyslogIdentifier" "$service"
     iniset -sudo $unitfile "Service" "User" "$user"
     iniset -sudo $unitfile "Service" "ExecStart" "$command"
@@ -1614,6 +1616,9 @@ function _run_under_systemd {
     fi
     local env_vars="$5"
     if [[ "$command" =~ "uwsgi" ]] ; then
+        if [[ "$GLOBAL_VENV" == "True" ]] ; then
+            cmd="$cmd --venv $DEVSTACK_VENV"
+        fi
         write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars"
     else
         write_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars"

inc/python

@@ -32,6 +32,23 @@ function join_extras {
 # Python Functions
 # ================
 
+# Setup the global devstack virtualenvs and the associated environment
+# updates.
+function setup_devstack_virtualenv {
+    # We run devstack out of a global virtualenv.
+    if [[ ! -d $DEVSTACK_VENV ]] ; then
+        # Using system site packages to enable nova to use libguestfs.
+        # This package is currently installed via the distro and not
+        # available on pypi.
+        python$PYTHON3_VERSION -m venv --system-site-packages $DEVSTACK_VENV
+        pip_install -U pip
+    fi
+    if [[ ":$PATH:" != *":$DEVSTACK_VENV/bin:"* ]] ; then
+        export PATH="$DEVSTACK_VENV/bin:$PATH"
+        export PYTHON="$DEVSTACK_VENV/bin/python3"
+    fi
+}
+
 # Get the path to the pip command.
 # get_pip_command
 function get_pip_command {
@@ -60,8 +77,11 @@ function get_python_exec_prefix {
     fi
     $xtrace
 
-    local PYTHON_PATH=/usr/local/bin
-    echo $PYTHON_PATH
+    if [[ "$GLOBAL_VENV" == "True" ]] ; then
+        echo "$DEVSTACK_VENV/bin"
+    else
+        echo "/usr/local/bin"
+    fi
 }
 
 # Wrapper for ``pip install`` that only installs versions of libraries
@@ -166,6 +186,14 @@ function pip_install {
     if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
         local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
         local sudo_pip="env"
+    elif [[ "${GLOBAL_VENV}" == "True" && -d ${DEVSTACK_VENV} ]] ; then
+        # We have to check that the DEVSTACK_VENV exists because early
+        # devstack boostrapping needs to operate in a system context
+        # too bootstrap pip. Once pip is bootstrapped we create the
+        # global venv and can start to use it.
+        local cmd_pip=$DEVSTACK_VENV/bin/pip
+        local sudo_pip="env"
+        echo "Using python $PYTHON3_VERSION to install $package_dir"
     else
         local cmd_pip="python$PYTHON3_VERSION -m pip"
         # See
@@ -439,7 +467,7 @@ function setup_package {
 
     pip_install $flags "$project_dir$extras"
     # ensure that further actions can do things like setup.py sdist
-    if [[ "$flags" == "-e" ]]; then
+    if [[ "$flags" == "-e" && "$GLOBAL_VENV" == "False" ]]; then
         safe_chown -R $STACK_USER $1/*.egg-info
     fi
 }

inc/rootwrap

@@ -60,6 +60,11 @@ function configure_rootwrap {
     sudo install -o root -g root -m 644 $rootwrap_conf_src_dir/rootwrap.conf /etc/${project}/rootwrap.conf
     sudo sed -e "s:^filters_path=.*$:filters_path=/etc/${project}/rootwrap.d:" -i /etc/${project}/rootwrap.conf
 
+    # Rely on $PATH set by devstack to determine what is safe to execute
+    # by rootwrap rather than use explicit whitelist of paths in
+    # rootwrap.conf
+    sudo sed -e 's/^exec_dirs=.*/#&/' -i /etc/${project}/rootwrap.conf
+
     # Set up the rootwrap sudoers
     local tempfile
     tempfile=$(mktemp)

lib/glance

@@ -47,6 +47,9 @@ USE_CINDER_FOR_GLANCE=$(trueorfalse False USE_CINDER_FOR_GLANCE)
 # from CINDER_ENABLED_BACKENDS
 GLANCE_CINDER_DEFAULT_BACKEND=${GLANCE_CINDER_DEFAULT_BACKEND:-lvmdriver-1}
 GLANCE_STORE_ROOTWRAP_BASE_DIR=/usr/local/etc/glance
+if [[ "$GLOBAL_VENV" == "True" ]] ; then
+    GLANCE_STORE_ROOTWRAP_BASE_DIR=${DEVSTACK_VENV}/etc/glance
+fi
 # When Cinder is used as a glance store, you can optionally configure cinder to
 # optimize bootable volume creation by allowing volumes to be cloned directly
 # in the backend instead of transferring data via Glance.  To use this feature,

lib/horizon

@@ -115,6 +115,11 @@ function configure_horizon {
     local horizon_conf
     horizon_conf=$(apache_site_config_for horizon)
 
+    local wsgi_venv_config=""
+    if [[ "$GLOBAL_VENV" == "True" ]] ; then
+        wsgi_venv_config="WSGIPythonHome $DEVSTACK_VENV"
+    fi
+
     # Configure apache to run horizon
     # Set up the django horizon application to serve via apache/wsgi
     sudo sh -c "sed -e \"
@@ -124,6 +129,7 @@ function configure_horizon {
         s,%APACHE_NAME%,$APACHE_NAME,g;
         s,%DEST%,$DEST,g;
         s,%WEBROOT%,$HORIZON_APACHE_ROOT,g;
+        s,%WSGIPYTHONHOME%,$wsgi_venv_config,g;
     \" $FILES/apache-horizon.template >$horizon_conf"
 
     if is_ubuntu; then

lib/tls

@@ -364,8 +364,11 @@ function deploy_int_CA {
 function fix_system_ca_bundle_path {
     if is_service_enabled tls-proxy; then
         local capath
-        capath=$(python3 -c $'try:\n from requests import certs\n print (certs.where())\nexcept ImportError: pass')
-
+        if [[ "$GLOBAL_VENV" == "True" ]] ; then
+            capath=$($DEVSTACK_VENV/bin/python3 -c $'try:\n from requests import certs\n print (certs.where())\nexcept ImportError: pass')
+        else
+            capath=$(python3 -c $'try:\n from requests import certs\n print (certs.where())\nexcept ImportError: pass')
+        fi
         if [[ ! $capath == "" && ! $capath =~ ^/etc/.* && ! -L $capath ]]; then
             if is_fedora; then
                 sudo rm -f $capath

stack.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+
 # ``stack.sh`` is an opinionated OpenStack developer installation.  It
 # installs and configures various combinations of **Cinder**, **Glance**,
 # **Horizon**, **Keystone**, **Nova**, **Neutron**, and **Swift**
@@ -824,6 +825,17 @@ fi
 source $TOP_DIR/tools/fixup_stuff.sh
 fixup_all
 
+if [[ "$GLOBAL_VENV" == "True" ]] ; then
+    # TODO(frickler): find a better solution for this
+    sudo ln -sf /opt/stack/data/venv/bin/privsep-helper /usr/local/bin
+    sudo ln -sf /opt/stack/data/venv/bin/cinder-rtstool /usr/local/bin
+    sudo ln -sf /opt/stack/data/venv/bin/openstack /usr/local/bin
+    sudo ln -sf /opt/stack/data/venv/bin/tox /usr/local/bin
+    sudo ln -sf /opt/stack/data/venv/bin/nova-manage /usr/local/bin
+
+    setup_devstack_virtualenv
+fi
+
 # Install subunit for the subunit output stream
 pip_install -U os-testr
 

stackrc

@@ -183,6 +183,14 @@ IDENTITY_API_VERSION=3
 # each services ${SERVICE}_ENFORCE_SCOPE variables
 ENFORCE_SCOPE=$(trueorfalse False ENFORCE_SCOPE)
 
+# Devstack supports the use of a global virtualenv. These variables enable
+# and disable this functionality as well as set the path to the virtualenv.
+# Note that the DATA_DIR is selected because grenade testing uses a shared
+# DATA_DIR but different DEST dirs and we don't want two sets of venvs,
+# instead we want one global set.
+GLOBAL_VENV=$(trueorfalse True GLOBAL_VENV)
+DEVSTACK_VENV=${DEVSTACK_VENV:-$DATA_DIR/venv}
+
 # Enable use of Python virtual environments.  Individual project use of
 # venvs are controlled by the PROJECT_VENV array; every project with
 # an entry in the array will be installed into the named venv.

tools/install_prereqs.sh

@@ -79,6 +79,8 @@ if [[ -n "$SYSLOG" && "$SYSLOG" != "False" ]]; then
     fi
 fi
 
+# TODO(clarkb) remove these once we are switched to global venv by default
+export PYTHON=$(which python${PYTHON3_VERSION} 2>/dev/null || which python3 2>/dev/null)
 
 # Mark end of run
 # ---------------

tools/memory_tracker.sh

@@ -14,7 +14,12 @@
 
 set -o errexit
 
-PYTHON=${PYTHON:-python3}
+# TODO(frickler): make this use stackrc variables
+if [ -x /opt/stack/data/venv/bin/python ]; then
+    PYTHON=/opt/stack/data/venv/bin/python
+else
+    PYTHON=${PYTHON:-python3}
+fi
 
 # time to sleep between checks
 SLEEP_TIME=20