Add GRENADE_USE_EXTERNAL_DEVSTACK to skip part of the setup

When GRENADE_USE_EXTERNAL_DEVSTACK is set to True,
the initial steps performed by grenade are skipped, namely:
- grabbing and configuring the base and the target devstacks;
- running devstacks on the base target.

This change is required to allow a native Zuul v3 job to use
the existing workflow to setup and run devstack.

Change-Id: I232db8de05141849c81851dd29440959cb0d8533

PLUGINS.rst

@@ -241,6 +241,12 @@ devstack localrc files with the ``devstack_localrc`` function.
 Which will take all the rest of the stuff on that line and add it to
 the localrc for either the base or target devstack.
 
+Please note that ``devstack_localrc`` only works when grenade
+performs the configuration of the devstack settings and runs devstack
+against the base target. When GRENADE_USE_EXTERNAL_DEVSTACK is set
+to True, as it happens on the Zuul grenade jobs where devstack is
+configured and executed before grenade, the function has no effect.
+
 Example settings
 ----------------
 

README.rst

@@ -108,6 +108,10 @@ high level version of what that does.
 - perform some sanity checking (currently tempest smoke) to ensure
   everything seems good.
 
+The script skips the first two steps (which take care of setting up the 2
+devstack environments and installing the base one) when the value
+of GRENADE_USE_EXTERNAL_DEVSTACK is set to True.
+
 
 Terminology
 -----------

grenade.sh

@@ -166,9 +166,13 @@ set -o xtrace
 # the TARGET devstack functions file, then source the rest of the
 # grenade settings. This should let us run the bulk of grenade.
 
-# Get both devstack trees, so that BASE_DEVSTACK_DIR, and
+if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then
-# TARGET_DEVSTACK_DIR are now fully populated.
+    # Get both devstack trees, so that BASE_DEVSTACK_DIR, and
-fetch_devstacks
+    # TARGET_DEVSTACK_DIR are now fully populated.
+    fetch_devstacks
+else
+    devstacks_setup_environment
+fi
 
 # Source the rest of the Grenade functions. For convenience
 # ``$GRENADE_DIR/functions`` implicitly sources
@@ -215,16 +219,17 @@ fetch_grenade_plugins
 # when the time is right.
 load_settings
 
+if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then
+    # Nova should use singleconductor as Grenade doesn't
+    # setup multi-cell rabbit for now
+    devstack_localrc base "CELLSV2_SETUP=singleconductor"
+    devstack_localrc target "CELLSV2_SETUP=singleconductor"
+fi
+
 # And ensure that we setup the target localrc.auto, because stack.sh
 # isn't run there. This has to be run after load_settings because
 # plugins might change the service list during this phase.
 
-# Nova should use singleconductor as Grenade doesn't
-# setup multi-cell rabbit for now
-devstack_localrc base "CELLSV2_SETUP=singleconductor"
-devstack_localrc target "CELLSV2_SETUP=singleconductor"
-
-
 extract_localrc_section $TARGET_DEVSTACK_DIR/local.conf \
                         $TARGET_DEVSTACK_DIR/localrc \
                         $TARGET_DEVSTACK_DIR/.localrc.auto
@@ -235,26 +240,28 @@ if [[ "$RUN_BASE" == "True" ]]; then
     # Initialize grenade_db local storage, used for resource tracking
     init_grenade_db
 
-    echo_summary "Running base stack.sh"
+    if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then
-    cd $BASE_DEVSTACK_DIR
+        echo_summary "Running base stack.sh"
-    GIT_BASE=$GIT_BASE ./stack.sh
+        cd $BASE_DEVSTACK_DIR
-    stop $STOP stack.sh 10
+        GIT_BASE=$GIT_BASE ./stack.sh
+        stop $STOP stack.sh 10
 
-    echo_summary "Running post-stack.sh"
+        echo_summary "Running post-stack.sh"
-    if [[ -e $GRENADE_DIR/post-stack.sh ]]; then
+        if [[ -e $GRENADE_DIR/post-stack.sh ]]; then
-        cd $GRENADE_DIR
+            cd $GRENADE_DIR
 
-        # By the time we get here the sub nodes are already setup with localrc files
+            # By the time we get here the sub nodes are already setup with localrc files
-        # as those are transferred in devstack-gate even before grenade.sh is called
+            # as those are transferred in devstack-gate even before grenade.sh is called
-        # We hack the ./post-stack.sh to inject what we need. if we don't set
+            # We hack the ./post-stack.sh to inject what we need. if we don't set
-        # CELLSV2_SETUP, the default devstack assumes "superconductor" and fails.
+            # CELLSV2_SETUP, the default devstack assumes "superconductor" and fails.
-        export SUB_NODE_ENV_VARS="CELLSV2_SETUP=singleconductor"
+            export SUB_NODE_ENV_VARS="CELLSV2_SETUP=singleconductor"
-        sed -i 's/stdbuf/$SUB_NODE_ENV_VARS stdbuf/' ./post-stack.sh
+            sed -i 's/stdbuf/$SUB_NODE_ENV_VARS stdbuf/' ./post-stack.sh
-        cat ./post-stack.sh
+            cat ./post-stack.sh
 
-        ./post-stack.sh
+            ./post-stack.sh
-        stop $STOP post-stack.sh 15
+            stop $STOP post-stack.sh 15
-        echo_summary "Completed post-stack.sh"
+            echo_summary "Completed post-stack.sh"
+        fi
     fi
 
     # Cache downloaded instances

grenaderc

@@ -8,6 +8,13 @@ if [ -f $RC_DIR/localrc ]; then
     source $RC_DIR/localrc
 fi
 
+# If True, the setting of the devstack configuration for the base and
+# the target and the execution of devstack on the base target
+# (if requested) are not performed by grenade, but they must be done
+# separately.
+# This is the case for example for the native zuul v3 grenade job.
+GRENADE_USE_EXTERNAL_DEVSTACK=${GRENADE_USE_EXTERNAL_DEVSTACK:-False}
+
 # Base GIT Repo URL
 # Another option is http://review.openstack.org/p
 # Another option is https://github.com

inc/bootstrap

@@ -261,6 +261,12 @@ function dump_local_files {
 }
 
 function fetch_devstacks {
+    devstacks_clone
+    devstacks_setup_environment
+    devstacks_setup_settings
+}
+
+function devstacks_clone {
     # Fetch Base Devstack
 
     # Get DevStack if it doesn't exist
@@ -271,7 +277,9 @@ function fetch_devstacks {
     if [[ ! -d $TARGET_DEVSTACK_DIR ]]; then
         git_clone $TARGET_DEVSTACK_REPO $TARGET_DEVSTACK_DIR $TARGET_DEVSTACK_BRANCH
     fi
+}
 
+function devstacks_setup_environment {
     # This depends on REQUIREMENTS_DIR being set in grenaderc, which
     # it needs to be to have gotten this far.
     source $TARGET_DEVSTACK_DIR/functions-common
@@ -283,12 +291,21 @@ function fetch_devstacks {
     _DEFAULT_PYTHON3_VERSION="$(_get_python_version python3)"
     export PYTHON3_VERSION=${PYTHON3_VERSION:-${_DEFAULT_PYTHON3_VERSION:-3.5}}
 
-    install_devstack_tools
-
     # Load up a copy of the downloaded images if not present
     if [[ -d ${STACK_ROOT}/images ]]; then
         rsync -a ${STACK_ROOT}/images/* $BASE_DEVSTACK_DIR/files
     fi
+}
+
+function devstacks_setup_settings {
+    # dsconf is required only by this function and devstack_localrc
+    # but only when the base is installed by grenade.
+    # Moreover, when the deployment of base is not performed by grenade,
+    # installing devstack-tools just before creating the resources also
+    # updates pbr (at least from rocky->master) leading to an exception
+    # in running services using pbr methods (for example nova-conductor)
+    # probably because they are still looking for the old version.
+    install_devstack_tools
 
     # Set up base localrc
 

inc/plugin

@@ -165,6 +165,10 @@ function enable_grenade_plugin {
 }
 
 function devstack_localrc {
+    if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" == "True" ]; then
+        echo "DevStack is configured externally, ignoring \$(devstack_localrc $@)"
+        return
+    fi
     local settings_file=$(caller | awk '{print $2}')
     local where=$1
     local path=$(localrc_path $where)