yatinkarel
28382fcf1e
legacy-tempest-neutron-full-stable deploy's stable/wallaby and is running on ubuntu-bionic(py3.6) and failing in tempest setup as tempest master no longer works with py3.6. Master is dropping py3.6 support, currently oslo.log has already dropped the support and it's latest release is not installable on py3.6. Pin Markusafe, as since version 2.1.0 it dropped module 'soft_unicode' which is required by Jinja2. Also update ansible version to 2.9.6 as of focal which should auto discover python as per distro. So let's switch this job to ubuntu-focal which is py3.8. Change-Id: I37887bf430b4bce2b9d2179b2d6dd6c050bb1768
752 lines
30 KiB
Bash
Executable File
752 lines
30 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Gate commits to several projects on a VM running those projects
|
|
# configured by devstack.
|
|
|
|
# Copyright (C) 2011-2013 OpenStack Foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
#
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Most of the work of this script is done in functions so that we may
|
|
# easily redirect their stdout / stderr to log files.
|
|
|
|
GIT_BASE=${GIT_BASE:-https://opendev.org}
|
|
GIT_BRANCH=${GIT_BRANCH:-master}
|
|
|
|
# We're using enough ansible specific features that it's extremely
|
|
# possible that new ansible releases can break us. As such we should
|
|
# be very deliberate about which ansible we use.
|
|
# NOTE(ykarel): Ansible 2.9.6 is current as of Ubuntu Focal 20.04.
|
|
# ARA is pinned to <1.0.0 below which affects the required version of Ansible.
|
|
ANSIBLE_VERSION=${ANSIBLE_VERSION:-2.9.6}
|
|
export DSTOOLS_VERSION=${DSTOOLS_VERSION:-0.4.0}
|
|
|
|
# Set to 0 to skip stackviz
|
|
export PROCESS_STACKVIZ=${PROCESS_STACKVIZ:-1}
|
|
|
|
# sshd may have been compiled with a default path excluding */sbin
|
|
export PATH=$PATH:/usr/local/sbin:/usr/sbin
|
|
# When doing xtrace (set -x / set -o xtrace), provide more debug output
|
|
export PS4='+ ${BASH_SOURCE:-}:${FUNCNAME[0]:-}:L${LINENO:-}: '
|
|
|
|
#check to see if WORKSPACE var is defined
|
|
if [ -z ${WORKSPACE} ]; then
|
|
echo "The 'WORKSPACE' variable is undefined. It must be defined for this script to work"
|
|
exit 1
|
|
fi
|
|
|
|
source $WORKSPACE/devstack-gate/functions.sh
|
|
|
|
start_timer
|
|
|
|
# Note that service/project enablement vars are here so that they can be
|
|
# used to select the PROJECTS list below reliably.
|
|
|
|
# Set to 1 to run sahara
|
|
export DEVSTACK_GATE_SAHARA=${DEVSTACK_GATE_SAHARA:-0}
|
|
|
|
# Set to 1 to run trove
|
|
export DEVSTACK_GATE_TROVE=${DEVSTACK_GATE_TROVE:-0}
|
|
|
|
# are we pulling any libraries from git
|
|
export DEVSTACK_PROJECT_FROM_GIT=${DEVSTACK_PROJECT_FROM_GIT:-}
|
|
|
|
# Save the PROJECTS variable as it was passed in. This is needed for reproduce.sh
|
|
# incase the job definition contains items that are not in the "global" list
|
|
# below.
|
|
# See: https://bugs.launchpad.net/openstack-gate/+bug/1544827
|
|
JOB_PROJECTS="$PROJECTS"
|
|
PROJECTS="openstack/devstack-gate $PROJECTS"
|
|
PROJECTS="openstack/devstack $PROJECTS"
|
|
PROJECTS="openstack/ceilometer $PROJECTS"
|
|
PROJECTS="openstack/ceilometermiddleware $PROJECTS"
|
|
PROJECTS="openstack/cinder $PROJECTS"
|
|
PROJECTS="openstack/glance $PROJECTS"
|
|
PROJECTS="openstack/heat $PROJECTS"
|
|
PROJECTS="openstack/heat-cfntools $PROJECTS"
|
|
PROJECTS="openstack/heat-templates $PROJECTS"
|
|
if [[ "$DEVSTACK_GATE_HORIZON" -eq "1" || "$DEVSTACK_PROJECT_FROM_GIT" = "manila-ui" ]] ; then
|
|
PROJECTS="openstack/horizon $PROJECTS"
|
|
PROJECTS="openstack/manila-ui $PROJECTS"
|
|
fi
|
|
PROJECTS="openstack/keystone $PROJECTS"
|
|
PROJECTS="openstack/neutron $PROJECTS"
|
|
PROJECTS="openstack/nova $PROJECTS"
|
|
PROJECTS="openstack/requirements $PROJECTS"
|
|
PROJECTS="openstack/swift $PROJECTS"
|
|
PROJECTS="openstack/tempest $PROJECTS"
|
|
# Everything below this line in the PROJECTS list is for non
|
|
# default devstack runs. Overtime we should remove items from
|
|
# below and add them explicitly to the jobs that need them. The
|
|
# reason for this is to reduce job runtimes, every git repo
|
|
# has to be cloned and updated and checked out to the proper ref
|
|
# which is not free.
|
|
PROJECTS="openstack/tripleo-ci $PROJECTS"
|
|
# The devstack heat plugin uses these repos
|
|
if [[ "$DEVSTACK_GATE_HEAT" -eq "1" ]] ; then
|
|
PROJECTS="openstack/dib-utils $PROJECTS"
|
|
PROJECTS="openstack/diskimage-builder $PROJECTS"
|
|
fi
|
|
PROJECTS="openstack/glance_store $PROJECTS"
|
|
PROJECTS="openstack/keystoneauth $PROJECTS"
|
|
PROJECTS="openstack/keystonemiddleware $PROJECTS"
|
|
PROJECTS="openstack/manila $PROJECTS"
|
|
PROJECTS="openstack/zaqar $PROJECTS"
|
|
PROJECTS="openstack/neutron-fwaas $PROJECTS"
|
|
PROJECTS="openstack/octavia $PROJECTS"
|
|
PROJECTS="openstack/neutron-vpnaas $PROJECTS"
|
|
PROJECTS="openstack/os-apply-config $PROJECTS"
|
|
PROJECTS="openstack/os-brick $PROJECTS"
|
|
PROJECTS="openstack/os-client-config $PROJECTS"
|
|
PROJECTS="openstack/os-collect-config $PROJECTS"
|
|
PROJECTS="openstack/os-net-config $PROJECTS"
|
|
PROJECTS="openstack/os-refresh-config $PROJECTS"
|
|
PROJECTS="openstack/osc-lib $PROJECTS"
|
|
if [[ "$DEVSTACK_GATE_SAHARA" -eq "1" ]] ; then
|
|
PROJECTS="openstack/sahara $PROJECTS"
|
|
PROJECTS="openstack/sahara-dashboard $PROJECTS"
|
|
fi
|
|
PROJECTS="openstack/tripleo-heat-templates $PROJECTS"
|
|
PROJECTS="openstack/tripleo-image-elements $PROJECTS"
|
|
if [[ "$DEVSTACK_GATE_TROVE" -eq "1" ]] ; then
|
|
PROJECTS="openstack/trove $PROJECTS"
|
|
fi
|
|
if [[ -n "$DEVSTACK_PROJECT_FROM_GIT" ]] ; then
|
|
# We populate the PROJECTS list with any libs that should be installed
|
|
# from source and not pypi assuming that live under openstack/
|
|
TRAILING_COMMA_REMOVED=$(echo "$DEVSTACK_PROJECT_FROM_GIT" | sed -e 's/,$//')
|
|
PROCESSED_FROM_GIT=$(echo "openstack/$TRAILING_COMMA_REMOVED" | sed -e 's/,/ openstack\//g')
|
|
PROJECTS="$PROCESSED_FROM_GIT $PROJECTS"
|
|
fi
|
|
|
|
# Include openstack/placement starting in Stein.
|
|
stable_compare="stable/[a-r]"
|
|
if [[ ! "$OVERRIDE_ZUUL_BRANCH" =~ $stable_compare ]] ; then
|
|
PROJECTS="openstack/placement $PROJECTS"
|
|
fi
|
|
|
|
# Remove duplicates as they result in errors when managing
|
|
# git state.
|
|
PROJECTS=$(echo $PROJECTS | tr '[:space:]' '\n' | sort -u)
|
|
|
|
echo "The PROJECTS list is:"
|
|
echo $PROJECTS | fold -w 80 -s
|
|
echo "---"
|
|
|
|
export BASE=/opt/stack
|
|
|
|
# The URL from which to fetch ZUUL references
|
|
export ZUUL_URL=${ZUUL_URL:-http://zuul.openstack.org/p}
|
|
|
|
# The feature matrix to select devstack-gate components
|
|
export DEVSTACK_GATE_FEATURE_MATRIX=${DEVSTACK_GATE_FEATURE_MATRIX:-roles/test-matrix/files/features.yaml}
|
|
|
|
# Set to 1 to install, configure and enable the Tempest test suite; more flags may be
|
|
# required to be set to customize the test run, e.g. DEVSTACK_GATE_TEMPEST_STRESS=1
|
|
export DEVSTACK_GATE_TEMPEST=${DEVSTACK_GATE_TEMPEST:-0}
|
|
|
|
# Set to 1, in conjunction with DEVSTACK_GATE_TEMPEST, will allow Tempest to be
|
|
# installed and configured, but the tests will be skipped
|
|
export DEVSTACK_GATE_TEMPEST_NOTESTS=${DEVSTACK_GATE_TEMPEST_NOTESTS:-0}
|
|
|
|
# Set to 1 to run postgresql instead of mysql
|
|
export DEVSTACK_GATE_POSTGRES=${DEVSTACK_GATE_POSTGRES:-0}
|
|
|
|
# Set to 1 to use zeromq instead of rabbitmq (or qpid)
|
|
export DEVSTACK_GATE_ZEROMQ=${DEVSTACK_GATE_ZEROMQ:-0}
|
|
|
|
# Set to qpid to use qpid, or zeromq to use zeromq.
|
|
# Default set to rabbitmq
|
|
export DEVSTACK_GATE_MQ_DRIVER=${DEVSTACK_GATE_MQ_DRIVER:-"rabbitmq"}
|
|
|
|
# This value must be provided when DEVSTACK_GATE_TEMPEST_STRESS is set.
|
|
export DEVSTACK_GATE_TEMPEST_STRESS_ARGS=${DEVSTACK_GATE_TEMPEST_STRESS_ARGS:-""}
|
|
|
|
# Set to 1 to run tempest heat slow tests
|
|
export DEVSTACK_GATE_TEMPEST_HEAT_SLOW=${DEVSTACK_GATE_TEMPEST_HEAT_SLOW:-0}
|
|
|
|
# Set to 1 to run tempest large ops test
|
|
export DEVSTACK_GATE_TEMPEST_LARGE_OPS=${DEVSTACK_GATE_TEMPEST_LARGE_OPS:-0}
|
|
|
|
# Set to 1 to run tempest smoke tests serially
|
|
export DEVSTACK_GATE_SMOKE_SERIAL=${DEVSTACK_GATE_SMOKE_SERIAL:-0}
|
|
|
|
# Set to 1 to explicitly disable tempest tenant isolation. Otherwise tenant isolation setting
|
|
# for tempest will be the one chosen by devstack.
|
|
export DEVSTACK_GATE_TEMPEST_DISABLE_TENANT_ISOLATION=${DEVSTACK_GATE_TEMPEST_DISABLE_TENANT_ISOLATION:-0}
|
|
|
|
# Should cinder perform secure deletion of volumes?
|
|
# Defaults to none to avoid bug 1023755. Can also be set to zero or shred.
|
|
# Only applicable to stable/liberty+ devstack.
|
|
export DEVSTACK_CINDER_VOLUME_CLEAR=${DEVSTACK_CINDER_VOLUME_CLEAR:-none}
|
|
|
|
# Set this to override the branch selected for testing (in
|
|
# single-branch checkouts; not used for grenade)
|
|
export OVERRIDE_ZUUL_BRANCH=${OVERRIDE_ZUUL_BRANCH:-$ZUUL_BRANCH}
|
|
|
|
stable_compare="stable/[a-n]"
|
|
|
|
# Set to 1 to run neutron instead of nova network
|
|
# This is a bit complicated to handle the deprecation of nova net across
|
|
# repos with branches from this branchless job runner.
|
|
if [ -n "$DEVSTACK_GATE_NEUTRON" ] ; then
|
|
# If someone has made a choice externally honor it
|
|
export DEVSTACK_GATE_NEUTRON=$DEVSTACK_GATE_NEUTRON
|
|
elif [[ "$OVERRIDE_ZUUL_BRANCH" =~ $stable_compare ]] ; then
|
|
# Default to no neutron on older stable branches because nova net
|
|
# was the default all that time.
|
|
export DEVSTACK_GATE_NEUTRON=0
|
|
else
|
|
# For everything else there is neutron
|
|
export DEVSTACK_GATE_NEUTRON=1
|
|
fi
|
|
|
|
|
|
# Set to 1 to run neutron distributed virtual routing
|
|
export DEVSTACK_GATE_NEUTRON_DVR=${DEVSTACK_GATE_NEUTRON_DVR:-0}
|
|
|
|
# This variable tells devstack-gate to set up an overlay network between the nodes.
|
|
export DEVSTACK_GATE_NET_OVERLAY=${DEVSTACK_GATE_NET_OVERLAY:-$DEVSTACK_GATE_NEUTRON_DVR}
|
|
|
|
# Set to 1 to run nova in cells mode instead of the default mode
|
|
export DEVSTACK_GATE_CELLS=${DEVSTACK_GATE_CELLS:-0}
|
|
|
|
# Set to 1 to run nova in with nova metadata server as a separate binary
|
|
export DEVSTACK_GATE_NOVA_API_METADATA_SPLIT=${DEVSTACK_GATE_NOVA_API_METADATA_SPLIT:-0}
|
|
|
|
# Set to 1 to run ironic baremetal provisioning service.
|
|
export DEVSTACK_GATE_IRONIC=${DEVSTACK_GATE_IRONIC:-0}
|
|
|
|
# Set to "agent_ipmitool" to run ironic with the ironic-python-agent driver
|
|
export DEVSTACK_GATE_IRONIC_DRIVER=${DEVSTACK_GATE_IRONIC_DRIVER:-pxe_ipmitool}
|
|
|
|
|
|
# Set to 0 to avoid building Ironic deploy ramdisks
|
|
export DEVSTACK_GATE_IRONIC_BUILD_RAMDISK=${DEVSTACK_GATE_IRONIC_BUILD_RAMDISK:-1}
|
|
|
|
# Set to 0 to disable config_drive and use the metadata server instead
|
|
export DEVSTACK_GATE_CONFIGDRIVE=${DEVSTACK_GATE_CONFIGDRIVE:-0}
|
|
|
|
# Set to 1 to enable installing test requirements
|
|
export DEVSTACK_GATE_INSTALL_TESTONLY=${DEVSTACK_GATE_INSTALL_TESTONLY:-0}
|
|
|
|
# Set the number of threads to run tempest with
|
|
DEFAULT_CONCURRENCY=$(nproc)
|
|
if [ ${DEFAULT_CONCURRENCY} -gt 3 ] ; then
|
|
DEFAULT_CONCURRENCY=$((${DEFAULT_CONCURRENCY} / 2))
|
|
fi
|
|
export TEMPEST_CONCURRENCY=${TEMPEST_CONCURRENCY:-${DEFAULT_CONCURRENCY}}
|
|
|
|
# The following variable is set for different directions of Grenade updating
|
|
# for a stable branch we want to both try to upgrade forward n => n+1 as
|
|
# well as upgrade from last n-1 => n.
|
|
#
|
|
# i.e. stable/ocata:
|
|
# pullup means stable/newton => stable/ocata
|
|
# forward means stable/ocata => master (or stable/pike if that's out)
|
|
export DEVSTACK_GATE_GRENADE=${DEVSTACK_GATE_GRENADE:-}
|
|
|
|
# the branch name for selecting grenade branches
|
|
GRENADE_BASE_BRANCH=${OVERRIDE_ZUUL_BRANCH:-${ZUUL_BRANCH}}
|
|
|
|
|
|
if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then
|
|
# All grenade upgrades get tempest
|
|
export DEVSTACK_GATE_TEMPEST=1
|
|
|
|
# NOTE(sdague): Adjusting grenade branches for a release.
|
|
#
|
|
# When we get to the point of the release where we should adjust
|
|
# the grenade branches, the order of doing so is important.
|
|
#
|
|
# 1. stable/foo on all projects in devstack
|
|
# 2. stable/foo on devstack
|
|
# 3. stable/foo on grenade
|
|
# 4. adjust branches in devstack-gate
|
|
#
|
|
# The devstack-gate branch logic going last means that it will be
|
|
# tested before thrust upon the jobs. For both the stable/kilo and
|
|
# stable/liberty releases real release issues were found in this
|
|
# process. So this should be done as early as possible.
|
|
|
|
case $DEVSTACK_GATE_GRENADE in
|
|
|
|
# sideways upgrades try to move between configurations in the
|
|
# same release, typically used for migrating between services
|
|
# or configurations.
|
|
sideways-*)
|
|
export GRENADE_OLD_BRANCH="$GRENADE_BASE_BRANCH"
|
|
export GRENADE_NEW_BRANCH="$GRENADE_BASE_BRANCH"
|
|
;;
|
|
|
|
# forward upgrades are an attempt to migrate up from an
|
|
# existing stable branch to the next release.
|
|
forward)
|
|
if [[ "$GRENADE_BASE_BRANCH" == "stable/kilo" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/kilo"
|
|
export GRENADE_NEW_BRANCH="stable/liberty"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/liberty" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/liberty"
|
|
export GRENADE_NEW_BRANCH="stable/mitaka"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/mitaka" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/mitaka"
|
|
export GRENADE_NEW_BRANCH="stable/newton"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/newton" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/newton"
|
|
export GRENADE_NEW_BRANCH="$GIT_BRANCH"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/ocata" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/ocata"
|
|
export GRENADE_NEW_BRANCH="stable/pike"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/pike" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/pike"
|
|
export GRENADE_NEW_BRANCH="stable/queens"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/queens" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/queens"
|
|
export GRENADE_NEW_BRANCH="stable/rocky"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/rocky" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/rocky"
|
|
export GRENADE_NEW_BRANCH="stable/stein"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/stein" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/stein"
|
|
export GRENADE_NEW_BRANCH="stable/train"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/train" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/train"
|
|
export GRENADE_NEW_BRANCH="stable/ussuri"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/ussuri" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/ussuri"
|
|
export GRENADE_NEW_BRANCH="stable/victoria"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/victoria" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/victoria"
|
|
export GRENADE_NEW_BRANCH="stable/wallaby"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/wallaby" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/wallaby"
|
|
export GRENADE_NEW_BRANCH="$GIT_BRANCH"
|
|
fi
|
|
;;
|
|
|
|
# pullup upgrades are our normal upgrade test. Can you upgrade
|
|
# to the current patch from the last stable.
|
|
pullup)
|
|
if [[ "$GRENADE_BASE_BRANCH" == "stable/liberty" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/kilo"
|
|
export GRENADE_NEW_BRANCH="stable/liberty"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/mitaka" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/liberty"
|
|
export GRENADE_NEW_BRANCH="stable/mitaka"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/newton" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/mitaka"
|
|
export GRENADE_NEW_BRANCH="stable/newton"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/ocata" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/newton"
|
|
export GRENADE_NEW_BRANCH="stable/ocata"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/pike" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/ocata"
|
|
export GRENADE_NEW_BRANCH="stable/pike"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/queens" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/pike"
|
|
export GRENADE_NEW_BRANCH="stable/queens"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/rocky" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/queens"
|
|
export GRENADE_NEW_BRANCH="stable/rocky"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/stein" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/rocky"
|
|
export GRENADE_NEW_BRANCH="stable/stein"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/train" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/stein"
|
|
export GRENADE_NEW_BRANCH="stable/train"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/ussuri" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/train"
|
|
export GRENADE_NEW_BRANCH="stable/ussuri"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/victoria" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/ussuri"
|
|
export GRENADE_NEW_BRANCH="stable/victoria"
|
|
elif [[ "$GRENADE_BASE_BRANCH" == "stable/wallaby" ]]; then
|
|
export GRENADE_OLD_BRANCH="stable/victoria"
|
|
export GRENADE_NEW_BRANCH="stable/wallaby"
|
|
else # master
|
|
export GRENADE_OLD_BRANCH="stable/wallaby"
|
|
export GRENADE_NEW_BRANCH="$GIT_BRANCH"
|
|
fi
|
|
;;
|
|
|
|
# If we got here, someone typoed a thing, and we should fail
|
|
# explicitly so they don't accidentally pass in some what that
|
|
# is unexpected.
|
|
*)
|
|
echo "Unsupported upgrade mode: $DEVSTACK_GATE_GRENADE"
|
|
exit 1
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
# Set the virtualization driver to: libvirt, openvz, xenapi
|
|
export DEVSTACK_GATE_VIRT_DRIVER=${DEVSTACK_GATE_VIRT_DRIVER:-libvirt}
|
|
|
|
# Use qemu by default for consistency since some providers enable
|
|
# nested virt
|
|
export DEVSTACK_GATE_LIBVIRT_TYPE=${DEVSTACK_GATE_LIBVIRT_TYPE:-qemu}
|
|
|
|
# See switch below for this -- it gets set to 1 when tempest
|
|
# is the project being gated.
|
|
export DEVSTACK_GATE_TEMPEST_FULL=${DEVSTACK_GATE_TEMPEST_FULL:-0}
|
|
|
|
# Set to 1 to run all tempest tests
|
|
export DEVSTACK_GATE_TEMPEST_ALL=${DEVSTACK_GATE_TEMPEST_ALL:-0}
|
|
|
|
# Set to 1 to run all tempest scenario tests
|
|
export DEVSTACK_GATE_TEMPEST_SCENARIOS=${DEVSTACK_GATE_TEMPEST_SCENARIOS:-0}
|
|
|
|
# Set to a regex to run tempest with a custom regex filter
|
|
export DEVSTACK_GATE_TEMPEST_REGEX=${DEVSTACK_GATE_TEMPEST_REGEX:-""}
|
|
|
|
# Set to 1 to run all-plugin tempest tests
|
|
export DEVSTACK_GATE_TEMPEST_ALL_PLUGINS=${DEVSTACK_GATE_TEMPEST_ALL_PLUGINS:-0}
|
|
|
|
# Set to 1 if running the openstack/requirements integration test
|
|
export DEVSTACK_GATE_REQS_INTEGRATION=${DEVSTACK_GATE_REQS_INTEGRATION:-0}
|
|
|
|
# Set to 0 to disable clean logs enforcement (3rd party CI might want to do this
|
|
# until they get their driver cleaned up)
|
|
export DEVSTACK_GATE_CLEAN_LOGS=${DEVSTACK_GATE_CLEAN_LOGS:-1}
|
|
|
|
# Set this to the time in milliseconds that the entire job should be
|
|
# allowed to run before being aborted (default 120 minutes=7200000ms).
|
|
# This may be supplied by Jenkins based on the configured job timeout
|
|
# which is why it's in this convenient unit.
|
|
export BUILD_TIMEOUT=$(expr ${BUILD_TIMEOUT:-7200000} / 60000)
|
|
|
|
# Set this to the time in minutes that should be reserved for
|
|
# uploading artifacts at the end after a timeout. Defaults to 10
|
|
# minutes.
|
|
export DEVSTACK_GATE_TIMEOUT_BUFFER=${DEVSTACK_GATE_TIMEOUT_BUFFER:-10}
|
|
|
|
# Not user servicable.
|
|
export DEVSTACK_GATE_TIMEOUT=$(expr $BUILD_TIMEOUT - $DEVSTACK_GATE_TIMEOUT_BUFFER)
|
|
|
|
# Set to 1 to remove the stack users blanket sudo permissions forcing
|
|
# openstack services running as the stack user to rely on rootwrap rulesets
|
|
# instead of raw sudo. Do this to ensure rootwrap works. This is the default.
|
|
export DEVSTACK_GATE_REMOVE_STACK_SUDO=${DEVSTACK_GATE_REMOVE_STACK_SUDO:-1}
|
|
|
|
# Set to 1 to unstack immediately after devstack installation. This
|
|
# is intended to be a stop-gap until devstack can support
|
|
# dependency-only installation.
|
|
export DEVSTACK_GATE_UNSTACK=${DEVSTACK_GATE_UNSTACK:-0}
|
|
|
|
# The topology of the system determinates the service distribution
|
|
# among the nodes.
|
|
# aio: `all in one` just only one node used
|
|
# aiopcpu: `all in one plus compute` one node will be installed as aio
|
|
# the extra nodes will gets only limited set of services
|
|
# ctrlpcpu: `controller plus compute` One node will gets the controller type
|
|
# services without the compute type of services, the others gets,
|
|
# the compute style services several services can be common,
|
|
# the networking services also presents on the controller [WIP]
|
|
export DEVSTACK_GATE_TOPOLOGY=${DEVSTACK_GATE_TOPOLOGY:-aio}
|
|
|
|
# Set to a space-separated list of projects to prepare in the
|
|
# workspace, e.g. 'openstack/devstack openstack/neutron'.
|
|
# Minimizing the number of targeted projects can reduce the setup cost
|
|
# for jobs that know exactly which repos they need.
|
|
export DEVSTACK_GATE_PROJECTS_OVERRIDE=${DEVSTACK_GATE_PROJECTS_OVERRIDE:-""}
|
|
|
|
# Set this to "True" to force devstack to pick python 3.x. "False" will cause
|
|
# devstack to pick python 2.x. We should leave this empty for devstack to
|
|
# pick the default.
|
|
export DEVSTACK_GATE_USE_PYTHON3=${DEVSTACK_GATE_USE_PYTHON3:-""}
|
|
|
|
# Set this to enable remote logging of the console via UDP packets to
|
|
# a specified ipv4 ip:port (note; not hostname -- ip address only).
|
|
# This can be extremely useful if a host is oopsing or dropping off
|
|
# the network amd you are not getting any useful logs from jenkins.
|
|
#
|
|
# To capture these logs, enable a netcat/socat type listener to
|
|
# capture UDP packets at the specified remote ip. For example:
|
|
#
|
|
# $ nc -v -u -l -p 6666 | tee save-output.log
|
|
# or
|
|
# $ socat udp-recv:6666 - | tee save-output.log
|
|
#
|
|
# One further trick is to send interesting data to /dev/ksmg; this
|
|
# data will get out over the netconsole even if the main interfaces
|
|
# have been disabled, etc. e.g.
|
|
#
|
|
# $ ip addr | sudo tee /dev/ksmg
|
|
#
|
|
export DEVSTACK_GATE_NETCONSOLE=${DEVSTACK_GATE_NETCONSOLE:-""}
|
|
enable_netconsole
|
|
|
|
if [ -n "$DEVSTACK_GATE_PROJECTS_OVERRIDE" ]; then
|
|
PROJECTS=$DEVSTACK_GATE_PROJECTS_OVERRIDE
|
|
fi
|
|
|
|
if ! function_exists "gate_hook"; then
|
|
# the command we use to run the gate
|
|
function gate_hook {
|
|
$BASE/new/devstack-gate/devstack-vm-gate.sh
|
|
}
|
|
export -f gate_hook
|
|
fi
|
|
|
|
echo "Triggered by: https://review.openstack.org/$ZUUL_CHANGE patchset $ZUUL_PATCHSET"
|
|
echo "Pipeline: $ZUUL_PIPELINE"
|
|
echo "Timeout set to $DEVSTACK_GATE_TIMEOUT minutes \
|
|
with $DEVSTACK_GATE_TIMEOUT_BUFFER minutes reserved for cleanup."
|
|
echo "Available disk space on this host:"
|
|
indent df -h
|
|
|
|
if command -v python3 &>/dev/null; then
|
|
PIP=pip3
|
|
PYTHON_VER=$(python3 -c 'import sys; print("%s.%s" % sys.version_info[0:2])')
|
|
else
|
|
PIP=pip
|
|
PYTHON_VER=2.7
|
|
fi
|
|
|
|
# Install ansible
|
|
|
|
# TODO(gmann): virtualenv 20.0.1 is broken, one known issue:
|
|
# https://github.com/pypa/virtualenv/issues/1551
|
|
# Once virtualenv is fixed we can use the latest one.
|
|
sudo -H $PIP install "virtualenv<20.0.0"
|
|
virtualenv -p python${PYTHON_VER} /tmp/ansible
|
|
|
|
# Explicitly install pbr first as this will use pip rathat than
|
|
# easy_install. Hope is this is generally more reliable.
|
|
/tmp/ansible/bin/pip install pbr
|
|
/tmp/ansible/bin/pip install ansible==$ANSIBLE_VERSION \
|
|
devstack-tools==$DSTOOLS_VERSION 'ara<1.0.0' 'cmd2<0.9.0' \
|
|
'flask<2.0.0' 'alembic<1.5.0' 'importlib-resources<5.1.3' \
|
|
'MarkupSafe<2.1.0'
|
|
export ANSIBLE=/tmp/ansible/bin/ansible
|
|
export ANSIBLE_PLAYBOOK=/tmp/ansible/bin/ansible-playbook
|
|
export ANSIBLE_CONFIG="$WORKSPACE/ansible.cfg"
|
|
export DSCONF=/tmp/ansible/bin/dsconf
|
|
|
|
# Write inventory file with groupings
|
|
COUNTER=1
|
|
PRIMARY_NODE=$(cat /etc/nodepool/primary_node_private)
|
|
echo "[primary]" > "$WORKSPACE/inventory"
|
|
echo "localhost ansible_connection=local host_counter=$COUNTER nodepool='{\"private_ipv4\": \"$PRIMARY_NODE\"}'" >> "$WORKSPACE/inventory"
|
|
echo "[subnodes]" >> "$WORKSPACE/inventory"
|
|
export SUBNODES=$(cat /etc/nodepool/sub_nodes_private)
|
|
for SUBNODE in $SUBNODES ; do
|
|
let COUNTER=COUNTER+1
|
|
echo "$SUBNODE host_counter=$COUNTER nodepool='{\"private_ipv4\": \"$SUBNODE\"}'" >> "$WORKSPACE/inventory"
|
|
done
|
|
|
|
# Write ansible config file
|
|
cat > $ANSIBLE_CONFIG <<EOF
|
|
[defaults]
|
|
callback_plugins = $WORKSPACE/devstack-gate/playbooks/plugins/callback:/tmp/ansible/lib/python${PYTHON_VER}/site-packages/ara/plugins/callbacks
|
|
stdout_callback = devstack
|
|
|
|
# Disable SSH host key checking
|
|
host_key_checking = False
|
|
EOF
|
|
|
|
# NOTE(clarkb): for simplicity we evaluate all bash vars in ansible commands
|
|
# on the node running these scripts, we do not pass through unexpanded
|
|
# vars to ansible shell commands. This may need to change in the future but
|
|
# for now the current setup is simple, consistent and easy to understand.
|
|
|
|
# This is in brackets for avoiding inheriting a huge environment variable
|
|
(export PROJECTS; export > "$WORKSPACE/test_env.sh")
|
|
# Copy bootstrap to remote hosts
|
|
$ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m copy \
|
|
-a "src='$WORKSPACE/devstack-gate' dest='$WORKSPACE'"
|
|
$ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m copy \
|
|
-a "src='$WORKSPACE/test_env.sh' dest='$WORKSPACE/test_env.sh'"
|
|
|
|
# Make a directory to store logs
|
|
$ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m file \
|
|
-a "path='$WORKSPACE/logs' state=absent"
|
|
$ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m file \
|
|
-a "path='$WORKSPACE/logs' state=directory"
|
|
|
|
# Record a file to reproduce this build
|
|
reproduce "$JOB_PROJECTS"
|
|
|
|
# Run ansible to do setup_host on all nodes.
|
|
echo "Setting up the hosts"
|
|
|
|
# This function handles any common exit paths from here on in
|
|
function exit_handler {
|
|
local status=$1
|
|
|
|
# Generate ARA report
|
|
/tmp/ansible/bin/ara generate html $WORKSPACE/logs/ara
|
|
gzip --recursive --best $WORKSPACE/logs/ara
|
|
|
|
if [[ $status -ne 0 ]]; then
|
|
echo "*** FAILED with status: $status"
|
|
else
|
|
echo "SUCCESSFULLY FINISHED"
|
|
fi
|
|
|
|
exit $status
|
|
}
|
|
|
|
# little helper that runs anything passed in under tsfilter
|
|
function run_command {
|
|
local fn="$@"
|
|
local cmd=""
|
|
|
|
# note that we want to keep the tsfilter separate; it's a trap for
|
|
# new-players that errexit isn't applied if we do "&& tsfilter
|
|
# ..." and thus we won't pick up any failures in the commands the
|
|
# function runs.
|
|
#
|
|
# Note we also send stderr to stdout, otherwise ansible consumes
|
|
# each separately and outputs them separately. That doesn't work
|
|
# well for log files; especially running "xtrace" in bash which
|
|
# puts tracing on stderr.
|
|
read -r -d '' cmd <<EOF
|
|
source '$WORKSPACE/test_env.sh'
|
|
source '$WORKSPACE/devstack-gate/functions.sh'
|
|
set -o errexit
|
|
tsfilter $fn 2>&1
|
|
executable=/bin/bash
|
|
EOF
|
|
|
|
echo "$cmd"
|
|
}
|
|
|
|
rc=0
|
|
|
|
echo "... this takes a few seconds (logs at logs/devstack-gate-setup-host.txt.gz)"
|
|
$ANSIBLE_PLAYBOOK -f 5 -i "$WORKSPACE/inventory" "$WORKSPACE/devstack-gate/playbooks/setup_host.yaml" \
|
|
&> "$WORKSPACE/logs/devstack-gate-setup-host.txt" || rc=$?
|
|
if [[ $rc -ne 0 ]]; then
|
|
exit_handler $rc;
|
|
fi
|
|
|
|
if [ -n "$DEVSTACK_GATE_GRENADE" ]; then
|
|
start=$(date +%s)
|
|
echo "Setting up the new (migrate to) workspace"
|
|
echo "... this takes 3 - 5 minutes (logs at logs/devstack-gate-setup-workspace-new.txt.gz)"
|
|
$ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell \
|
|
-a "$(run_command setup_workspace '$GRENADE_NEW_BRANCH' '$BASE/new')" \
|
|
&> "$WORKSPACE/logs/devstack-gate-setup-workspace-new.txt" || rc=$?
|
|
if [[ $rc -ne 0 ]]; then
|
|
exit_handler $rc;
|
|
fi
|
|
echo "Setting up the old (migrate from) workspace ..."
|
|
echo "... this takes 3 - 5 minutes (logs at logs/devstack-gate-setup-workspace-old.txt.gz)"
|
|
$ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell \
|
|
-a "$(run_command setup_workspace '$GRENADE_OLD_BRANCH' '$BASE/old')" \
|
|
&> "$WORKSPACE/logs/devstack-gate-setup-workspace-old.txt" || rc=$?
|
|
end=$(date +%s)
|
|
took=$((($end - $start) / 60))
|
|
if [[ "$took" -gt 20 ]]; then
|
|
echo "WARNING: setup of 2 workspaces took > 20 minutes, this is a very slow node."
|
|
fi
|
|
if [[ $rc -ne 0 ]]; then
|
|
exit_handler $rc;
|
|
fi
|
|
else
|
|
echo "Setting up the workspace"
|
|
echo "... this takes 3 - 5 minutes (logs at logs/devstack-gate-setup-workspace-new.txt.gz)"
|
|
start=$(date +%s)
|
|
$ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell \
|
|
-a "$(run_command setup_workspace '$OVERRIDE_ZUUL_BRANCH' '$BASE/new')" \
|
|
&> "$WORKSPACE/logs/devstack-gate-setup-workspace-new.txt" || rc=$?
|
|
end=$(date +%s)
|
|
took=$((($end - $start) / 60))
|
|
if [[ "$took" -gt 10 ]]; then
|
|
echo "WARNING: setup workspace took > 10 minutes, this is a very slow node."
|
|
fi
|
|
if [[ $rc -ne 0 ]]; then
|
|
exit_handler $rc;
|
|
fi
|
|
fi
|
|
|
|
# relocate and symlink logs into $BASE to save space on the root filesystem
|
|
# TODO: make this more ansibley
|
|
$ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell -a "
|
|
if [ -d '$WORKSPACE/logs' -a \! -e '$BASE/logs' ]; then
|
|
sudo mv '$WORKSPACE/logs' '$BASE/'
|
|
ln -s '$BASE/logs' '$WORKSPACE/'
|
|
fi executable=/bin/bash"
|
|
|
|
# The DEVSTACK_GATE_SETTINGS variable may contain a path to a script that
|
|
# should be sourced after the environment has been set up. This is useful for
|
|
# allowing projects to provide a script in their repo that sets some custom
|
|
# environment variables.
|
|
check_for_devstack_gate_settings() {
|
|
if [ -f $1 ] ; then
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
if [ -n "${DEVSTACK_GATE_SETTINGS}" ] ; then
|
|
if check_for_devstack_gate_settings ${DEVSTACK_GATE_SETTINGS} ; then
|
|
source ${DEVSTACK_GATE_SETTINGS}
|
|
else
|
|
echo "WARNING: DEVSTACK_GATE_SETTINGS file does not exist: '${DEVSTACK_GATE_SETTINGS}'"
|
|
fi
|
|
fi
|
|
|
|
# Note that hooks should be multihost aware if necessary.
|
|
# devstack-vm-gate-wrap.sh will not automagically run the hooks on each node.
|
|
# Run pre test hook if we have one
|
|
with_timeout call_hook_if_defined "pre_test_hook"
|
|
GATE_RETVAL=$?
|
|
if [ $GATE_RETVAL -ne 0 ]; then
|
|
echo "ERROR: the pre-test setup script run by this job failed - exit code: $GATE_RETVAL"
|
|
fi
|
|
|
|
# Run the gate function
|
|
if [ $GATE_RETVAL -eq 0 ]; then
|
|
echo "Running gate_hook"
|
|
with_timeout "gate_hook"
|
|
GATE_RETVAL=$?
|
|
if [ $GATE_RETVAL -ne 0 ]; then
|
|
echo "ERROR: the main setup script run by this job failed - exit code: $GATE_RETVAL"
|
|
fi
|
|
fi
|
|
RETVAL=$GATE_RETVAL
|
|
|
|
if [ $GATE_RETVAL -ne 0 ]; then
|
|
echo " please look at the relevant log files to determine the root cause"
|
|
echo "Running devstack worlddump.py"
|
|
sudo $BASE/new/devstack/tools/worlddump.py -d $BASE/logs
|
|
fi
|
|
|
|
# Run post test hook if we have one
|
|
if [ $GATE_RETVAL -eq 0 ]; then
|
|
# Run post_test_hook if we have one
|
|
with_timeout call_hook_if_defined "post_test_hook"
|
|
RETVAL=$?
|
|
fi
|
|
|
|
if [ $GATE_RETVAL -eq 137 ] && [ -f $WORKSPACE/gate.pid ] ; then
|
|
echo "Job timed out"
|
|
GATEPID=`cat $WORKSPACE/gate.pid`
|
|
echo "Killing process group ${GATEPID}"
|
|
sudo kill -s 9 -${GATEPID}
|
|
fi
|
|
|
|
echo "Cleaning up host"
|
|
echo "... this takes 3 - 4 minutes (logs at logs/devstack-gate-cleanup-host.txt.gz)"
|
|
$ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell \
|
|
-a "$(run_command cleanup_host)" &> "$WORKSPACE/devstack-gate-cleanup-host.txt"
|
|
$ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m synchronize \
|
|
-a "mode=pull src='$BASE/logs/' dest='$BASE/logs/subnode-{{ host_counter }}' copy_links=yes"
|
|
sudo mv $WORKSPACE/devstack-gate-cleanup-host.txt $BASE/logs/
|
|
|
|
exit_handler $RETVAL
|