From d8dd053b0added2ef7952a247484ca950083fcd3 Mon Sep 17 00:00:00 2001 From: Paul Michali Date: Fri, 26 Jun 2015 17:39:16 +0000 Subject: [PATCH] VPNaaS: Don't clone neutron automatically for tests As part of the work to allow VPN functionals job to run when Neutron commits are reviewed, we need to be more intelligent about how dependencies are resolved in the tox.ini for this repo. This commit does the same as LBaaS, using a script to determine whether an existing Neutron repo exists (and use that for a Neutron patch-set running the VPN tests) or whether the Neutron repo needs to be cloned (for a VPN patch set). With the VPN repo, the rootwrap config file and filters need to be set up. The deploy_rootwrap script is modified to do all actions related to setup, instead of doing part of that in tox.ini, as before. In addition, rootwrap config and needed filters for testing from Neutron are added to this repo, so there is no dependency on Neutron (whose location cannot be assumed in the gate). NOTE: To run functional tests locally, one needs to set VENV for the desired functional target and invoke configure_vpn_for_func_testing.sh. Next, run tox for the functional target with --notest to setup the virtual environment. Then, install Neutron into the virtual env. Finally, the functional test can then be invoked. This must be upstreamed, before allowing the VPN functional job to run for Neutron jobs. Change-Id: Idb3c4dff0dbb40e30b2b49591df0b014b769f167 --- .../contrib/functional-test-rootwrap.conf | 34 ++++++++++ .../tests/contrib/functional-testing.filters | 63 +++++++++++++++++++ tools/deploy_rootwrap.sh | 58 +++++++++++++++++ tools/tox_install.sh | 41 ++++++++++++ tox.ini | 21 +++---- 5 files changed, 204 insertions(+), 13 deletions(-) create mode 100644 neutron_vpnaas/tests/contrib/functional-test-rootwrap.conf create mode 100644 neutron_vpnaas/tests/contrib/functional-testing.filters create mode 100755 tools/deploy_rootwrap.sh create mode 100755 tools/tox_install.sh diff --git a/neutron_vpnaas/tests/contrib/functional-test-rootwrap.conf b/neutron_vpnaas/tests/contrib/functional-test-rootwrap.conf new file mode 100644 index 000000000..f2d9ce422 --- /dev/null +++ b/neutron_vpnaas/tests/contrib/functional-test-rootwrap.conf @@ -0,0 +1,34 @@ +# Configuration for neutron-rootwrap +# This file should be owned by (and only-writeable by) the root user + +[DEFAULT] +# List of directories to load filter definitions from (separated by ','). +# These directories MUST all be only writeable by root ! +filters_path=/etc/neutron/rootwrap.d,/usr/share/neutron/rootwrap + +# List of directories to search executables in, in case filters do not +# explicitely specify a full path (separated by ',') +# If not specified, defaults to system PATH environment variable. +# These directories MUST all be only writeable by root ! +exec_dirs=/sbin,/usr/sbin,/bin,/usr/bin,/usr/local/bin + +# Enable logging to syslog +# Default value is False +use_syslog=False + +# Which syslog facility to use. +# Valid values include auth, authpriv, syslog, local0, local1... +# Default value is 'syslog' +syslog_log_facility=syslog + +# Which messages to log. +# INFO means log all usage +# ERROR means only log unsuccessful attempts +syslog_log_level=ERROR + +[xenapi] +# XenAPI configuration is only required by the L2 agent if it is to +# target a XenServer/XCP compute host's dom0. +xenapi_connection_url= +xenapi_connection_username=root +xenapi_connection_password= diff --git a/neutron_vpnaas/tests/contrib/functional-testing.filters b/neutron_vpnaas/tests/contrib/functional-testing.filters new file mode 100644 index 000000000..f4cb74cd5 --- /dev/null +++ b/neutron_vpnaas/tests/contrib/functional-testing.filters @@ -0,0 +1,63 @@ +# neutron-rootwrap command filters to support functional testing. It +# is NOT intended to be used outside of a test environment. +# +# This file should be owned by (and only-writeable by) the root user + +[Filters] +# enable ping from namespace +ping_filter: CommandFilter, ping, root +ping6_filter: CommandFilter, ping6, root + +# enable curl from namespace +curl_filter: CommandFilter, curl, root +tee_filter: CommandFilter, tee, root +tee_kill: KillFilter, root, tee, -9 +nc_filter: CommandFilter, nc, root +# netcat has different binaries depending on linux distribution +nc_kill: KillFilter, root, nc, -9 +ncbsd_kill: KillFilter, root, nc.openbsd, -9 +ncat_kill: KillFilter, root, ncat, -9 +ss_filter: CommandFilter, ss, root + +# arping +arping: CommandFilter, arping, root + +# l3_agent +sysctl: CommandFilter, sysctl, root +route: CommandFilter, route, root +radvd: CommandFilter, radvd, root + +# metadata proxy +metadata_proxy: CommandFilter, neutron-ns-metadata-proxy, root +# RHEL invocation of the metadata proxy will report /usr/bin/python +kill_metadata: KillFilter, root, python, -9 +kill_metadata7: KillFilter, root, python2.7, -9 +kill_radvd_usr: KillFilter, root, /usr/sbin/radvd, -9, -HUP +kill_radvd: KillFilter, root, /sbin/radvd, -9, -HUP + +# ip_lib +ip: IpFilter, ip, root +find: RegExpFilter, find, root, find, /sys/class/net, -maxdepth, 1, -type, l, -printf, %.* +ip_exec: IpNetnsExecFilter, ip, root + +# For ip monitor +kill_ip_monitor: KillFilter, root, ip, -9 + +# ovs_lib (if OVSInterfaceDriver is used) +ovs-vsctl: CommandFilter, ovs-vsctl, root + +# iptables_manager +iptables-save: CommandFilter, iptables-save, root +iptables-restore: CommandFilter, iptables-restore, root +ip6tables-save: CommandFilter, ip6tables-save, root +ip6tables-restore: CommandFilter, ip6tables-restore, root + +# Keepalived +keepalived: CommandFilter, keepalived, root +kill_keepalived: KillFilter, root, /usr/sbin/keepalived, -HUP, -15, -9 + +# l3 agent to delete floatingip's conntrack state +conntrack: CommandFilter, conntrack, root + +# keepalived state change monitor +keepalived_state_change: CommandFilter, neutron-keepalived-state-change, root diff --git a/tools/deploy_rootwrap.sh b/tools/deploy_rootwrap.sh new file mode 100755 index 000000000..00da99c2e --- /dev/null +++ b/tools/deploy_rootwrap.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -eu + +if [ $# -ne 2 ]; then + >&2 echo "Usage: $0 /path/to/repo /path/to/virtual-env +Deploy rootwrap configuration and filters. + +Warning: Any existing rootwrap files at the specified etc path will be +removed by this script. + +Optional: set OS_SUDO_TESTING=1 to deploy the filters required by +Neutron's functional testing suite." + exit 1 +fi + +OS_SUDO_TESTING=${OS_SUDO_TESTING:-0} + +repo_path=$1 +venv_path=$2 + +src_conf_path=${repo_path}/neutron_vpnaas/tests/contrib +src_conf=${src_conf_path}/functional-test-rootwrap.conf +src_rootwrap_path=${repo_path}/etc/neutron/rootwrap.d + +dst_conf_path=${venv_path}/etc/neutron +dst_conf=${dst_conf_path}/rootwrap.conf +dst_rootwrap_path=${dst_conf_path}/rootwrap.d + +# Clear any existing filters in virtual env +if [[ -d "$dst_rootwrap_path" ]]; then + rm -rf ${dst_rootwrap_path} +fi +mkdir -p -m 755 ${dst_rootwrap_path} + +# Get all needed filters +cp -p ${src_rootwrap_path}/* ${dst_rootwrap_path}/ +if [[ "$OS_SUDO_TESTING" = "1" ]]; then + cp -p ${repo_path}/neutron_vpnaas/tests/contrib/functional-testing.filters \ + ${dst_rootwrap_path}/ +fi +# Get config file and modify for this repo +cp -p ${src_conf} ${dst_conf} +sed -i "s:^filters_path=.*$:filters_path=${dst_rootwrap_path}:" ${dst_conf} +sed -i "s:^\(exec_dirs=.*\)$:\1,${venv_path}/bin:" ${dst_conf} +sudo cp ${dst_conf} /etc/neutron/ diff --git a/tools/tox_install.sh b/tools/tox_install.sh new file mode 100755 index 000000000..832dd55b1 --- /dev/null +++ b/tools/tox_install.sh @@ -0,0 +1,41 @@ +#!/bin/sh + +# Many of neutron's repos suffer from the problem of depending on neutron, +# but it not existing on pypi. + +# This wrapper for tox's package installer will use the existing package +# if it exists, else use zuul-cloner if that program exists, else grab it +# from neutron master via a hard-coded URL. That last case should only +# happen with devs running unit tests locally. + +# From the tox.ini config page: +# install_command=ARGV +# default: +# pip install {opts} {packages} + +ZUUL_CLONER=/usr/zuul-env/bin/zuul-cloner +neutron_installed=$(python -c "import neutron" ; echo $?) + +set -e + +if [ $neutron_installed -eq 0 ]; then + echo "ALREADY INSTALLED" > /tmp/tox_install.txt + echo "Neutron already installed; using existing package" +elif [ -x "$ZUUL_CLONER" ]; then + echo "ZUUL CLONER" > /tmp/tox_install.txt + cwd=$(/bin/pwd) + cd /tmp + $ZUUL_CLONER --cache-dir \ + /opt/git \ + git://git.openstack.org \ + openstack/neutron + cd openstack/neutron + pip install -e . + cd "$cwd" +else + echo "PIP HARDCODE" > /tmp/tox_install.txt + pip install -U -egit+https://git.openstack.org/openstack/neutron#egg=neutron +fi + +pip install -U $* +exit $? diff --git a/tox.ini b/tox.ini index 79f8277ee..4745d9555 100644 --- a/tox.ini +++ b/tox.ini @@ -6,9 +6,8 @@ skipsdist = True [testenv] setenv = VIRTUAL_ENV={envdir} usedevelop = True -install_command = pip install -U {opts} {packages} -deps = -egit+https://git.openstack.org/openstack/neutron#egg=neutron - -r{toxinidir}/requirements.txt +install_command = {toxinidir}/tools/tox_install.sh {opts} {packages} +deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt whitelist_externals = sh commands = @@ -16,10 +15,10 @@ commands = # there is also secret magic in pretty_tox.sh which lets you run in a fail only # mode. To do this define the TRACE_FAILONLY environmental variable. -[testenv:functional] -setenv = OS_TEST_PATH=./neutron-vpnaas/tests/functional -commands = - python setup.py testr --slowest --testr-args='{posargs}' +# [testenv:functional] +# setenv = OS_TEST_PATH=./neutron-vpnaas/tests/functional +# commands = +# python setup.py testr --slowest --testr-args='{posargs}' [testenv:dsvm-functional] setenv = OS_TEST_PATH=./neutron_vpnaas/tests/functional/openswan @@ -33,9 +32,7 @@ whitelist_externals = cp sudo commands = - {envdir}/src/neutron/tools/deploy_rootwrap.sh {envdir}/src/neutron {envdir}/etc {envdir}/bin - cp {toxinidir}/etc/neutron/rootwrap.d/vpnaas.filters {envdir}/etc/neutron/rootwrap.d/ - sudo cp {envdir}/etc/neutron/rootwrap.conf /etc/neutron/ + {toxinidir}/tools/deploy_rootwrap.sh {toxinidir} {envdir} sh tools/pretty_tox.sh '{posargs}' [testenv:dsvm-functional-sswan] @@ -50,9 +47,7 @@ whitelist_externals = cp sudo commands = - {envdir}/src/neutron/tools/deploy_rootwrap.sh {envdir}/src/neutron {envdir}/etc {envdir}/bin - cp {toxinidir}/etc/neutron/rootwrap.d/vpnaas.filters {envdir}/etc/neutron/rootwrap.d/ - sudo cp {envdir}/etc/neutron/rootwrap.conf /etc/neutron/ + {toxinidir}/tools/deploy_rootwrap.sh {toxinidir} {envdir} sh tools/pretty_tox.sh '{posargs}' [tox:jenkins]