VPNaaS: Revise functional test hooks

This commit does several things to allow functional tests to run, using a
DevStack configuration, but without stacking.

First, the gate_hook.sh is modified to specified environment variables
needed for processing, and will call configure_vpn_for_func_testing.sh.

The new configure_for_vpn_func_testing.sh script will first call the Neutron
configure_for_func_testing.sh script (as modified in [1]) to setup DevStack.
The script then installs the IPSec package, based on the functional job, and
will create the vpn_agent.ini file, in case it is needed for tests.

Minor changes are made to the post-test_hook.sh to indicate the location for
the Neutron and VPN repos.

Lastly, tox.ini is modified to place the rootwrap related files into the
virtual environment, copy in the VPN rootwrap filter, set up the rootwrap
commands, and copy the (customized) rootwrap.conf to /etc/neutron.

This last step is needed, because the neutron-vpn-netns-wrapper script that
is (currently) used for StrongSwan, uses /etc/neutron/rootwrap.conf and
doesn't allow overriding. This copying is a simple approach that will work
when multiple functional jobs are running, as the same config is used.

As a future follow-up commit, we could look into modifying the VPN device
drivers to pass the full path to the rootwrap.conf as a argument to the
neutron-vpn-netns-wrapper script (--rootwrap_config). The setting could
be added to the vpnagent.ini config file, available via config, to allow
this to be customized to something other than /etc/neutron (for production
use), and the functional tests could override it to point to the right
virtual environment.

To configure the environment for local run of the functional test, one
can do:
    tools/configure_for_vpn_func_testing.sh <devstack-repo> [-i]

where:
   devstack-repo... Path to devstack repo (/opt/stack/new/devstack
                    when run by gate)
   -i.............. Indicates to install Neutron package dependencies

For StrongSwan, prefix the command with VENV=dsvm-functional-sswan

Ref: [1] https://review.openstack.org/#/c/176064/

Change-Id: I2243b06b3cdbb352dac7bc47d03c830efb87be3c
Closes-Bug: #1445052
Depends-On: Ie490e89c1a65e126ad0a57d062076e472762837e

neutron_vpnaas/tests/contrib/gate_hook.sh

@@ -2,60 +2,26 @@
 
 set -ex
 
-CONTRIB_DIR="$BASE/new/neutron-vpnaas/neutron_vpnaas/tests/contrib"
+VENV=${1:-"dsvm-functional"}
 
-# Have DevStack use StrongSwan (instead of OpenSwan) for VPNaaS
-if [[ "$1" == "dsvm-functional-sswan" ]]; then
-    DEVSTACK_LOCAL_CONFIG=$'IPSEC_PACKAGE=strongswan\n'
-    export DEVSTACK_LOCAL_CONFIG
-    export KEEP_LOCALRC=1
-fi
+case $VENV in
+    dsvm-functional | dsvm-functional-sswan)
+        # The following need to be set before sourcing
+        # configure_for_func_testing.
+        GATE_DEST=$BASE/new
+        GATE_STACK_USER=stack
+        NEUTRON_PATH=$GATE_DEST/neutron
+        PROJECT_NAME=neutron-vpnaas
+        NEUTRON_VPN_PATH=$GATE_DEST/$PROJECT_NAME
+        DEVSTACK_PATH=$GATE_DEST/devstack
+        IS_GATE=True
 
-$BASE/new/devstack-gate/devstack-vm-gate.sh
+        source $NEUTRON_VPN_PATH/tools/configure_for_vpn_func_testing.sh
 
-# Add a rootwrap filter to support test-only
-# configuration (e.g. a KillFilter for processes that
-# use the python installed in a tox env).
-FUNC_FILTER=$CONTRIB_DIR/filters.template
-sed -e "s+\$BASE_PATH+$BASE/new/neutron-vpnaas/.tox/dsvm-functional+" \
-    $FUNC_FILTER | sudo tee /etc/neutron/rootwrap.d/functional.filters > /dev/null
-
-# Use devstack functions to install mysql and psql servers
-TOP_DIR=$BASE/new/devstack
-source $TOP_DIR/functions
-source $TOP_DIR/inc/meta-config
-source $TOP_DIR/stackrc
-source $TOP_DIR/lib/database
-source $TOP_DIR/localrc
-
-disable_service postgresql
-enable_service mysql
-initialize_database_backends
-install_database
-
-disable_service mysql
-enable_service postgresql
-initialize_database_backends
-install_database
-
-# Set up the 'openstack_citest' user and database in each backend
-tmp_dir=`mktemp -d`
-
-cat << EOF > $tmp_dir/mysql.sql
-CREATE DATABASE openstack_citest;
-CREATE USER 'openstack_citest'@'localhost' IDENTIFIED BY 'openstack_citest';
-CREATE USER 'openstack_citest' IDENTIFIED BY 'openstack_citest';
-GRANT ALL PRIVILEGES ON *.* TO 'openstack_citest'@'localhost';
-GRANT ALL PRIVILEGES ON *.* TO 'openstack_citest';
-FLUSH PRIVILEGES;
-EOF
-/usr/bin/mysql -u root < $tmp_dir/mysql.sql
-
-cat << EOF > $tmp_dir/postgresql.sql
-CREATE USER openstack_citest WITH CREATEDB LOGIN PASSWORD 'openstack_citest';
-CREATE DATABASE openstack_citest WITH OWNER openstack_citest;
-EOF
-# User/group postgres needs to be given access to tmp_dir
-setfacl -m g:postgres:rwx $tmp_dir
-sudo -u postgres /usr/bin/psql --file=$tmp_dir/postgresql.sql
+        # Make the workspace owned by the stack user
+        sudo chown -R $STACK_USER:$STACK_USER $BASE
 
+        configure_host_for_vpn_func_testing
+        ;;
+    api) $BASE/new/devstack-gate/devstack-vm-gate.sh ;;
+esac

neutron_vpnaas/tests/contrib/post_test_hook.sh

@@ -2,7 +2,7 @@
 
 set -xe
 
-NEUTRON_DIR="$BASE/new/neutron-vpnaas"
+NEUTRON_VPNAAS_DIR="$BASE/new/neutron-vpnaas"
 TEMPEST_DIR="$BASE/new/tempest"
 SCRIPTS_DIR="/usr/local/jenkins/slave_scripts"
 
@@ -21,23 +21,25 @@ function generate_testr_results {
     fi
 }
 
-function dsvm_functional_prep_func {
-    :
-}
-
+if [[ "$venv" == "dsvm-functional" || "$venv" == "dsvm-functional-sswan" ]]
+then
     owner=stack
-prep_func="dsvm_functional_prep_func"
+    sudo_env=
+elif [ "$venv" == "api" ]
+then
+    owner=tempest
+    # Configure the api tests to use the tempest.conf set by devstack.
+    sudo_env="TEMPEST_CONFIG_DIR=$TEMPEST_DIR/etc"
+fi
 
 # Set owner permissions according to job's requirements.
-cd $NEUTRON_DIR
-sudo chown -R $owner:stack $NEUTRON_DIR
-# Prep the environment according to job's requirements.
-$prep_func
+cd $NEUTRON_VPNAAS_DIR
+sudo chown -R $owner:stack $NEUTRON_VPNAAS_DIR
 
 # Run tests
-echo "Running neutron dsvm-functional test suite"
+echo "Running neutron $venv test suite"
 set +e
-sudo -H -u $owner tox -e $venv
+sudo -H -u $owner $sudo_env tox -e $venv
 testr_exit_code=$?
 set -e
 

tools/configure_for_vpn_func_testing.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+set -e
+
+
+IS_GATE=${IS_GATE:-False}
+PROJECT_NAME=${PROJECT_NAME:-neutron-vpnaas}
+REPO_BASE=${GATE_DEST:-$(cd $(dirname "$BASH_SOURCE")/../.. && pwd)}
+
+source $REPO_BASE/neutron/tools/configure_for_func_testing.sh
+
+
+function _install_vpn_package {
+    if [ "$VENV" == "dsvm-functional-sswan" ]
+    then
+        IPSEC_PACKAGE=strongswan
+    else
+        IPSEC_PACKAGE=openswan
+    fi
+
+    echo_summary "Installing $IPSEC_PACKAGE"
+    neutron_vpn_install_agent_packages
+}
+
+
+function _configure_vpn_ini_file {
+    echo_summary "Configuring VPN ini file"
+
+    local temp_ini=$(mktemp)
+    cp $REPO_BASE/$PROJECT_NAME/etc/vpn_agent.ini $temp_ini
+    if [ "$IPSEC_PACKAGE" == "strongswan" ]; then
+        iniset_multiline $temp_ini vpnagent vpn_device_driver neutron_vpnaas.services.vpn.device_drivers.strongswan_ipsec.StrongSwanDriver
+        if is_fedora; then
+            iniset $temp_ini strongswan default_config_area /usr/share/strongswan/templates/config/strongswan.d
+        fi
+    else
+        iniset_multiline $temp_ini vpnagent vpn_device_driver neutron_vpnaas.services.vpn.device_drivers.ipsec.OpenSwanDriver
+    fi
+    sudo install -d -o $STACK_USER /etc/neutron/
+    sudo mv $temp_ini $Q_VPN_CONF_FILE
+}
+
+
+function configure_host_for_vpn_func_testing {
+    echo_summary "Configuring for VPN functional testing"
+    if [ "$IS_GATE" == "True" ]; then
+        configure_host_for_func_testing
+    fi
+    _install_vpn_package
+    _configure_vpn_ini_file
+}
+
+
+if [ "$IS_GATE" != "True" ]; then
+    configure_host_for_vpn_func_testing
+fi
+

tox.ini

@@ -33,20 +33,36 @@ commands =
 [testenv:dsvm-functional]
 setenv = OS_TEST_PATH=./neutron_vpnaas/tests/functional/openswan
          OS_SUDO_TESTING=1
-         OS_ROOTWRAP_CMD=sudo /usr/local/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
+         OS_ROOTWRAP_CMD=sudo {envdir}/bin/neutron-rootwrap {envdir}/etc/neutron/rootwrap.conf
+         OS_ROOTWRAP_DAEMON_CMD=sudo {envdir}/bin/neutron-rootwrap-daemon {envdir}/etc/neutron/rootwrap.conf
          OS_FAIL_ON_MISSING_DEPS=1
 sitepackages=True
+whitelist_externals =
+  sh
+  cp
+  sudo
 commands =
-  python setup.py testr --slowest --testr-args='{posargs}'
+  {envdir}/src/neutron/tools/deploy_rootwrap.sh {envdir}/src/neutron {envdir}/etc {envdir}/bin
+  cp {toxinidir}/etc/neutron/rootwrap.d/vpnaas.filters {envdir}/etc/neutron/rootwrap.d/
+  sudo cp {envdir}/etc/neutron/rootwrap.conf /etc/neutron/
+  sh tools/pretty_tox.sh '{posargs}'
 
 [testenv:dsvm-functional-sswan]
 setenv = OS_TEST_PATH=./neutron_vpnaas/tests/functional/strongswan
          OS_SUDO_TESTING=1
-         OS_ROOTWRAP_CMD=sudo /usr/local/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
+         OS_ROOTWRAP_CMD=sudo {envdir}/bin/neutron-rootwrap {envdir}/etc/neutron/rootwrap.conf
+         OS_ROOTWRAP_DAEMON_CMD=sudo {envdir}/bin/neutron-rootwrap-daemon {envdir}/etc/neutron/rootwrap.conf
          OS_FAIL_ON_MISSING_DEPS=1
 sitepackages=True
+whitelist_externals =
+  sh
+  cp
+  sudo
 commands =
-  python setup.py testr --slowest --testr-args='{posargs}'
+  {envdir}/src/neutron/tools/deploy_rootwrap.sh {envdir}/src/neutron {envdir}/etc {envdir}/bin
+  cp {toxinidir}/etc/neutron/rootwrap.d/vpnaas.filters {envdir}/etc/neutron/rootwrap.d/
+  sudo cp {envdir}/etc/neutron/rootwrap.conf /etc/neutron/
+  sh tools/pretty_tox.sh '{posargs}'
 
 [tox:jenkins]
 sitepackages = True