VPNaaS: Don't clone neutron automatically for tests

As part of the work to allow VPN functionals job to run when Neutron
commits are reviewed, we need to be more intelligent about how
dependencies are resolved in the tox.ini for this repo.

This commit does the same as LBaaS, using a script to determine whether
an existing Neutron repo exists (and use that for a Neutron patch-set
running the VPN tests) or whether the Neutron repo needs to be cloned
(for a VPN patch set).

With the VPN repo, the rootwrap config file and filters need to be set
up. The deploy_rootwrap script is modified to do all actions related to
setup, instead of doing part of that in tox.ini, as before. In addition,
rootwrap config and needed filters for testing from Neutron are added to
this repo, so there is no dependency on Neutron (whose location cannot
be assumed in the gate).

NOTE: To run functional tests locally, one needs to set VENV for the
desired functional target and invoke configure_vpn_for_func_testing.sh.
Next, run tox for the functional target with --notest to setup the
virtual environment. Then, install Neutron into the virtual env. Finally,
the functional test can then be invoked.

This must be upstreamed, before allowing the VPN functional job to run
for Neutron jobs.

Change-Id: Idb3c4dff0dbb40e30b2b49591df0b014b769f167

neutron_vpnaas/tests/contrib/functional-test-rootwrap.conf

@@ -0,0 +1,34 @@
+# Configuration for neutron-rootwrap
+# This file should be owned by (and only-writeable by) the root user
+
+[DEFAULT]
+# List of directories to load filter definitions from (separated by ',').
+# These directories MUST all be only writeable by root !
+filters_path=/etc/neutron/rootwrap.d,/usr/share/neutron/rootwrap
+
+# List of directories to search executables in, in case filters do not
+# explicitely specify a full path (separated by ',')
+# If not specified, defaults to system PATH environment variable.
+# These directories MUST all be only writeable by root !
+exec_dirs=/sbin,/usr/sbin,/bin,/usr/bin,/usr/local/bin
+
+# Enable logging to syslog
+# Default value is False
+use_syslog=False
+
+# Which syslog facility to use.
+# Valid values include auth, authpriv, syslog, local0, local1...
+# Default value is 'syslog'
+syslog_log_facility=syslog
+
+# Which messages to log.
+# INFO means log all usage
+# ERROR means only log unsuccessful attempts
+syslog_log_level=ERROR
+
+[xenapi]
+# XenAPI configuration is only required by the L2 agent if it is to
+# target a XenServer/XCP compute host's dom0.
+xenapi_connection_url=<None>
+xenapi_connection_username=root
+xenapi_connection_password=<None>

neutron_vpnaas/tests/contrib/functional-testing.filters

@@ -0,0 +1,63 @@
+# neutron-rootwrap command filters to support functional testing.  It
+# is NOT intended to be used outside of a test environment.
+#
+# This file should be owned by (and only-writeable by) the root user
+
+[Filters]
+# enable ping from namespace
+ping_filter: CommandFilter, ping, root
+ping6_filter: CommandFilter, ping6, root
+
+# enable curl from namespace
+curl_filter: CommandFilter, curl, root
+tee_filter: CommandFilter, tee, root
+tee_kill: KillFilter, root, tee, -9
+nc_filter: CommandFilter, nc, root
+# netcat has different binaries depending on linux distribution
+nc_kill: KillFilter, root, nc, -9
+ncbsd_kill: KillFilter, root, nc.openbsd, -9
+ncat_kill: KillFilter, root, ncat, -9
+ss_filter: CommandFilter, ss, root
+
+# arping
+arping: CommandFilter, arping, root
+
+# l3_agent
+sysctl: CommandFilter, sysctl, root
+route: CommandFilter, route, root
+radvd: CommandFilter, radvd, root
+
+# metadata proxy
+metadata_proxy: CommandFilter, neutron-ns-metadata-proxy, root
+# RHEL invocation of the metadata proxy will report /usr/bin/python
+kill_metadata: KillFilter, root, python, -9
+kill_metadata7: KillFilter, root, python2.7, -9
+kill_radvd_usr: KillFilter, root, /usr/sbin/radvd, -9, -HUP
+kill_radvd: KillFilter, root, /sbin/radvd, -9, -HUP
+
+# ip_lib
+ip: IpFilter, ip, root
+find: RegExpFilter, find, root, find, /sys/class/net, -maxdepth, 1, -type, l, -printf, %.*
+ip_exec: IpNetnsExecFilter, ip, root
+
+# For ip monitor
+kill_ip_monitor: KillFilter, root, ip, -9
+
+# ovs_lib (if OVSInterfaceDriver is used)
+ovs-vsctl: CommandFilter, ovs-vsctl, root
+
+# iptables_manager
+iptables-save: CommandFilter, iptables-save, root
+iptables-restore: CommandFilter, iptables-restore, root
+ip6tables-save: CommandFilter, ip6tables-save, root
+ip6tables-restore: CommandFilter, ip6tables-restore, root
+
+# Keepalived
+keepalived: CommandFilter, keepalived, root
+kill_keepalived: KillFilter, root, /usr/sbin/keepalived, -HUP, -15, -9
+
+# l3 agent to delete floatingip's conntrack state
+conntrack: CommandFilter, conntrack, root
+
+# keepalived state change monitor
+keepalived_state_change: CommandFilter, neutron-keepalived-state-change, root

tools/deploy_rootwrap.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+set -eu
+
+if [ $# -ne 2 ]; then
+  >&2 echo "Usage: $0 /path/to/repo /path/to/virtual-env
+Deploy rootwrap configuration and filters.
+
+Warning: Any existing rootwrap files at the specified etc path will be
+removed by this script.
+
+Optional: set OS_SUDO_TESTING=1 to deploy the filters required by
+Neutron's functional testing suite."
+  exit 1
+fi
+
+OS_SUDO_TESTING=${OS_SUDO_TESTING:-0}
+
+repo_path=$1
+venv_path=$2
+
+src_conf_path=${repo_path}/neutron_vpnaas/tests/contrib
+src_conf=${src_conf_path}/functional-test-rootwrap.conf
+src_rootwrap_path=${repo_path}/etc/neutron/rootwrap.d
+
+dst_conf_path=${venv_path}/etc/neutron
+dst_conf=${dst_conf_path}/rootwrap.conf
+dst_rootwrap_path=${dst_conf_path}/rootwrap.d
+
+# Clear any existing filters in virtual env
+if [[ -d "$dst_rootwrap_path" ]]; then
+    rm -rf ${dst_rootwrap_path}
+fi
+mkdir -p -m 755 ${dst_rootwrap_path}
+
+# Get all needed filters
+cp -p ${src_rootwrap_path}/* ${dst_rootwrap_path}/
+if [[ "$OS_SUDO_TESTING" = "1" ]]; then
+    cp -p ${repo_path}/neutron_vpnaas/tests/contrib/functional-testing.filters \
+        ${dst_rootwrap_path}/
+fi
+# Get config file and modify for this repo
+cp -p ${src_conf} ${dst_conf}
+sed -i "s:^filters_path=.*$:filters_path=${dst_rootwrap_path}:" ${dst_conf}
+sed -i "s:^\(exec_dirs=.*\)$:\1,${venv_path}/bin:" ${dst_conf}
+sudo cp ${dst_conf} /etc/neutron/

tools/tox_install.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+
+# Many of neutron's repos suffer from the problem of depending on neutron,
+# but it not existing on pypi.
+
+# This wrapper for tox's package installer will use the existing package
+# if it exists, else use zuul-cloner if that program exists, else grab it
+# from neutron master via a hard-coded URL. That last case should only
+# happen with devs running unit tests locally.
+
+# From the tox.ini config page:
+# install_command=ARGV
+# default:
+# pip install {opts} {packages}
+
+ZUUL_CLONER=/usr/zuul-env/bin/zuul-cloner
+neutron_installed=$(python -c "import neutron" ; echo $?)
+
+set -e
+
+if [ $neutron_installed -eq 0 ]; then
+    echo "ALREADY INSTALLED" > /tmp/tox_install.txt
+    echo "Neutron already installed; using existing package"
+elif [ -x "$ZUUL_CLONER" ]; then
+    echo "ZUUL CLONER" > /tmp/tox_install.txt
+    cwd=$(/bin/pwd)
+    cd /tmp
+    $ZUUL_CLONER --cache-dir \
+        /opt/git \
+        git://git.openstack.org \
+        openstack/neutron
+    cd openstack/neutron
+    pip install -e .
+    cd "$cwd"
+else
+    echo "PIP HARDCODE" > /tmp/tox_install.txt
+    pip install -U -egit+https://git.openstack.org/openstack/neutron#egg=neutron
+fi
+
+pip install -U $*
+exit $?

tox.ini

@@ -6,9 +6,8 @@ skipsdist = True
 [testenv]
 setenv = VIRTUAL_ENV={envdir}
 usedevelop = True
-install_command = pip install -U {opts} {packages}
+install_command = {toxinidir}/tools/tox_install.sh {opts} {packages}
-deps = -egit+https://git.openstack.org/openstack/neutron#egg=neutron
+deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 whitelist_externals = sh
 commands =
@@ -16,10 +15,10 @@ commands =
 # there is also secret magic in pretty_tox.sh which lets you run in a fail only
 # mode. To do this define the TRACE_FAILONLY environmental variable.
 
-[testenv:functional]
+# [testenv:functional]
-setenv = OS_TEST_PATH=./neutron-vpnaas/tests/functional
+# setenv = OS_TEST_PATH=./neutron-vpnaas/tests/functional
-commands =
+# commands =
-  python setup.py testr --slowest --testr-args='{posargs}'
+#   python setup.py testr --slowest --testr-args='{posargs}'
 
 [testenv:dsvm-functional]
 setenv = OS_TEST_PATH=./neutron_vpnaas/tests/functional/openswan
@@ -33,9 +32,7 @@ whitelist_externals =
   cp
   sudo
 commands =
-  {envdir}/src/neutron/tools/deploy_rootwrap.sh {envdir}/src/neutron {envdir}/etc {envdir}/bin
+  {toxinidir}/tools/deploy_rootwrap.sh {toxinidir} {envdir}
-  cp {toxinidir}/etc/neutron/rootwrap.d/vpnaas.filters {envdir}/etc/neutron/rootwrap.d/
-  sudo cp {envdir}/etc/neutron/rootwrap.conf /etc/neutron/
   sh tools/pretty_tox.sh '{posargs}'
 
 [testenv:dsvm-functional-sswan]
@@ -50,9 +47,7 @@ whitelist_externals =
   cp
   sudo
 commands =
-  {envdir}/src/neutron/tools/deploy_rootwrap.sh {envdir}/src/neutron {envdir}/etc {envdir}/bin
+  {toxinidir}/tools/deploy_rootwrap.sh {toxinidir} {envdir}
-  cp {toxinidir}/etc/neutron/rootwrap.d/vpnaas.filters {envdir}/etc/neutron/rootwrap.d/
-  sudo cp {envdir}/etc/neutron/rootwrap.conf /etc/neutron/
   sh tools/pretty_tox.sh '{posargs}'
 
 [tox:jenkins]