openstack/trove-integration
Code Issues Proposed changes
5b3b1efe25
trove-integration/scripts/redstack
Matt Van Dijk 405a3b5452 Use Neutron by default 
nova-network is being deprecated, and devstack is moving to using
Neutron by default. This patch makes redstack use Neutron by default and
fixed a number of issues related to using Neutron.

Change-Id: If216688ef26b118c6d7598b2e086ba446b03978a
Closes-bug: 1604019
2016-08-18 15:17:15 +00:00

1402 lines
53 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
###############################################################################
# RedStack, the Trove Dev Machine Controller #
###############################################################################
# #
# This script provides all the functionality to run all the steps from #
# setting up the environment, resetting the nova database to running the #
# test. #
# #
###############################################################################
REDSTACK_SCRIPTS=${REDSTACK_SCRIPTS:-`pwd`}
REDSTACK_TESTS=$REDSTACK_SCRIPTS/../tests/
DEFAULT_LOCAL_CONF=local.conf.rc
DEFAULT_LOCALRC=localrc.rc
LOCAL_CONF=local.conf
LOCALRC=localrc
LOCALRC_AUTO=.localrc.auto
USER_LOCAL_CONF_NAME=.devstack.$LOCAL_CONF
USERHOME=$HOME
# Load options not checked into VCS.
if [ -f $USERHOME/.redstack.options.rc ]; then
. $USERHOME/.redstack.options.rc
fi
if [ -f $REDSTACK_SCRIPTS/options.rc ]; then
. $REDSTACK_SCRIPTS/options.rc
fi
# NOTE(mriedem): The gate-trove-functional-dsvm-* job config in project-config
# sets this value for Jenkins runs.
BRANCH_OVERRIDE=${BRANCH_OVERRIDE:-default}
if [[ $BRANCH_OVERRIDE == "default" && $OVERRIDE_ZUUL_BRANCH != "master" ]]; then
BRANCH_OVERRIDE=$OVERRIDE_ZUUL_BRANCH
fi
# Bail on errors.
set -e
# Get default host ip from interface
function get_default_host_ip() {
host_iface=$(ip route | sed -n '/^default/{ s/.*dev \(\w\+\)\s\+.*/\1/; p; }' | head -1)
echo `LC_ALL=C ip -f inet addr show ${host_iface} | awk '/inet/ {split($2,parts,"/"); print parts[1]}' | head -1`
}
# Load functions devstack style
. $REDSTACK_SCRIPTS/functions
. $REDSTACK_SCRIPTS/functions_qemu
# Load global configuration variables.
. $REDSTACK_SCRIPTS/redstack.rc
. $REDSTACK_SCRIPTS/reviews.rc
# allow overrides from devstack if already set
[[ -f $PATH_DEVSTACK_SRC/functions-common ]] && source $PATH_DEVSTACK_SRC/functions-common
[[ -f $PATH_DEVSTACK_SRC/functions ]] && source $PATH_DEVSTACK_SRC/functions
# Source the old-style localrc, or new-style .local.auto - only one should exist.
# Note: The devstack localrc's have references to 'enable_plugin' which causes
# errors when sourcing them in the stable/juno and stable/kilo branches.
# These errors are safe to ignore when sourcing these files.
set +e
[[ -f $PATH_DEVSTACK_SRC/$LOCALRC ]] && source $PATH_DEVSTACK_SRC/$LOCALRC
[[ -f $PATH_DEVSTACK_SRC/$LOCALRC_AUTO ]] && source $PATH_DEVSTACK_SRC/$LOCALRC_AUTO
set -e
# Set up variables for the CONF files - this has to happen after loading redstack.rc, since
# TROVE_CONF_DIR is defined there - these will be used by devstack too
export TROVE_CONF=$TROVE_CONF_DIR/trove.conf
export TROVE_TASKMANAGER_CONF=$TROVE_CONF_DIR/trove-taskmanager.conf
export TROVE_CONDUCTOR_CONF=$TROVE_CONF_DIR/trove-conductor.conf
export TROVE_GUESTAGENT_CONF=$TROVE_CONF_DIR/trove-guestagent.conf
export TROVE_API_PASTE_INI=$TROVE_CONF_DIR/api-paste.ini
export TEST_CONF=$TROVE_CONF_DIR/test.conf
# Public facing bits
SERVICE_PROTOCOL=${SERVICE_PROTOCOL:-http}
NETWORK_INTERFACE=${NETWORK_INTERFACE:-eth0}
NETWORK_SUBNET=${NETWORK_SUBNET:-10.0.0.0/24}
NETWORK_GATEWAY=${NETWORK_GATEWAY:-10.0.0.1}
BRIDGE_IP=${BRIDGE_IP:-172.24.4.1}
KEYSTONE_AUTH_HOST=${KEYSTONE_AUTH_HOST:-$SERVICE_HOST}
KEYSTONE_AUTH_PROTOCOL=${KEYSTONE_AUTH_PROTOCOL:-$SERVICE_PROTOCOL}
KEYSTONE_AUTH_PORT=${KEYSTONE_AUTH_PORT:-35357}
GLANCE_HOSTPORT=${GLANCE_HOSTPORT:-$SERVICE_HOST:9292}
GLANCE_SERVICE_PROTOCOL=${GLANCE_SERVICE_PROTOCOL:-http}
TROVE_AUTH_ENDPOINT=$KEYSTONE_AUTH_PROTOCOL://$KEYSTONE_AUTH_HOST:$KEYSTONE_AUTH_PORT/v2.0
# The following depends on whether neutron is used or nova-network
# neutron uses a bridge, nova-network does not
[[ $ENABLE_NEUTRON = true ]] && CONTROLLER_IP=$BRIDGE_IP || CONTROLLER_IP=$NETWORK_GATEWAY
# PATH_TROVE more than likely has file separators, which sed does not like
# This will escape them
ESCAPED_PATH_TROVE=$(echo $PATH_TROVE | sed 's/\//\\\//g')
ESCAPED_REDSTACK_SCRIPTS=$(echo $REDSTACK_SCRIPTS | sed 's/\//\\\//g')
TROVE_AUTH_CACHE_DIR=${TROVE_AUTH_CACHE_DIR:-/var/cache/trove}
TROVE_LOGDIR=${TROVE_LOGDIR:-$DEST/logs}
# DATASTORE_PKG_LOCATION defines the location from where the datastore packages
# can be accessed by the DIB elements. This is applicable only for datastores
# that do not have a public repository from where their packages can be accessed.
# This can either be a url to a private repository or a location on the local
# filesystem that contains the datastore packages.
DATASTORE_PKG_LOCATION=${DATASTORE_PKG_LOCATION:-}
# Support entry points installation of console scripts
if [[ -d $PATH_TROVE/bin ]]; then
TROVE_BIN_DIR=$PATH_TROVE/bin
else
TROVE_BIN_DIR=$(get_python_exec_prefix)
fi
# Allow sourcing config values from env.rc for overrides
if [ -f /tmp/integration/env.rc ]; then
source /tmp/integration/env.rc
fi
# set up respective package managers
if is_fedora; then
PKG_INSTALL_OPTS=""
PKG_MGR=dnf
PKG_GET_ARGS="-y"
else
PKG_INSTALL_OPTS="DEBIAN_FRONTEND=noninteractive"
PKG_MGR=apt-get
PKG_GET_ARGS="-y --allow-unauthenticated --force-yes"
fi
PKG_INSTALL_ARG="install"
PKG_UPDATE_ARG="update"
###############################################################################
# Utility functions
###############################################################################
# Colors that can be used in 'exclaim'
COLOR_RED='\033[0;31m'
COLOR_GREEN='\033[0;32m'
COLOR_BLUE='\033[0;34m'
COLOR_NONE='\033[0m'
function exclaim () {
echo "*******************************************************************************"
echo -e "$@"
echo "*******************************************************************************"
}
# Set the location of the Trove setup commands file for devstack - either in
# the devstack repo, or the Trove one. Also sets the flag which activates the
# devstack trove plugin, if required. We'll use the devstack version if it
# exists, otherwise we assume the plugin method.
function set_trove_plugin_vars () {
FAIL_IF_MISSING=${1:-true}
TROVE_SETUP_CMD_FILE="$PATH_DEVSTACK_SRC/lib/trove"
if [ -f "$TROVE_SETUP_CMD_FILE" ]; then
USE_DEVSTACK_TROVE_PLUGIN=false
else
TROVE_SETUP_CMD_FILE="$DEST/trove/devstack/plugin.sh"
USE_DEVSTACK_TROVE_PLUGIN=true
fi
# Only complain if we have a devstack directory and are told to
if [ "$FAIL_IF_MISSING" = "true" ] && [ -d "$PATH_DEVSTACK_SRC" ] && [ ! -f "$TROVE_SETUP_CMD_FILE" ]; then
exclaim "${COLOR_RED}Trove setup file '${TROVE_SETUP_CMD_FILE}' not found!${COLOR_NONE}"
exit 1
fi
}
function pkg_install () {
echo Installing $@...
sudo -E $PKG_INSTALL_OPTS $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS $PKG_INSTALL_ARG $@
}
function pkg_update () {
echo Updating $@...
sudo -E $PKG_INSTALL_OPTS $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS $PKG_UPDATE_ARG $@
}
function set_home_dir() {
exclaim "set_home_dir has been nooped."
exit 1
}
function set_http_proxy() {
if [ ! "${http_proxy}" = '' ]; then
HTTP_PROXY="http_proxy=$http_proxy https_proxy=$https_proxy"
fi
}
function get_ip_for_device() {
/sbin/ifconfig $1 | awk '/inet addr/{gsub(/addr:/,"");print $2}'
}
function ip_chunk() {
# Given 1-4 returns a bit of where the ip range starts.
# Full IP= `ip_chunk 1`.`ip_chunk 2`.`ip_chunk 3`.`ip_chunk 4`
get_ip_for_device $1 | cut -d. -f$2
}
# Add a flavor and a corresponding flavor.resize
# (flavor.resize adds 16 to the memory and one more vcpu)
function add_flavor() {
FLAVOR_NAME=$1
FLAVOR_ID=$2
FLAVOR_MEMORY_MB=$3
FLAVOR_ROOT_GB=$4
FLAVOR_VCPUS=$5
credentials="--os-username=admin --os-password=$ADMIN_PASSWORD --os-tenant-name=admin --os-auth-url=$TROVE_AUTH_ENDPOINT"
base_id=${FLAVOR_ID}
base_name_prefix=test
ephemeral_name_prefix=${base_name_prefix}.eph
for name_prefix in $base_name_prefix $ephemeral_name_prefix; do
reg_name=${name_prefix}.${FLAVOR_NAME}-${FLAVOR_ROOT_GB}
resize_name=${reg_name}.resize
ephemeral=0
if [[ $name_prefix == $ephemeral_name_prefix ]]; then
ephemeral=1
fi
for name in ${reg_name} ${resize_name}; do
id=$base_id
memory=${FLAVOR_MEMORY_MB}
vcpus=${FLAVOR_VCPUS}
if [[ $ephemeral != 0 ]]; then
if [[ $BRANCH_OVERRIDE == "stable/juno" || $BRANCH_OVERRIDE == "stable/kilo" ]]; then
id=1${id}
else
id=${id}e
fi
fi
if [[ $name == ${resize_name} ]]; then
if [[ $BRANCH_OVERRIDE == "stable/juno" || $BRANCH_OVERRIDE == "stable/kilo" ]]; then
id=2${id}
else
id=${id}r
fi
memory=$((${FLAVOR_MEMORY_MB} + 16))
vcpus=$((${FLAVOR_VCPUS} + 1))
fi
if [[ -z $(nova $credentials flavor-list | grep "| $name[ ]* |") ]]; then
nova $credentials flavor-create $name $id $memory $FLAVOR_ROOT_GB $vcpus --ephemeral $ephemeral
fi
done
done
}
function get_attribute_id() {
openstack --os-auth-url=$TROVE_AUTH_ENDPOINT --os-username=admin --os-password=$ADMIN_PASSWORD --os-project-name=admin $1 list | grep " $2" | get_field $3
}
###############################################################################
# Install all the required dependencies
###############################################################################
function install_prep_packages() {
# Called before devstack
exclaim 'Updating dependencies (part 1a)...'
pkg_update
exclaim 'Installing dependencies (part 1b)...'
pkg_install python-pip
if is_fedora; then
pkg_install git gettext
else
pkg_install git-core kvm-ipxe gettext
fi
sudo -H $HTTP_PROXY pip install --upgrade pip dib-utils
}
function install_devstack_code() {
exclaim "Installing devstack..."
# Installs devstack (if needed).
if [ ! -d $PATH_DEVSTACK_SRC ]; then
echo "DevStack not in a shared folder, cloning from git."
mkdir -p $PATH_DEVSTACK_SRC
git clone $DEVSTACK_REPO $PATH_DEVSTACK_SRC
fi
source $PATH_DEVSTACK_SRC/functions-common
source $PATH_DEVSTACK_SRC/functions
# Switch to a branch if specified. The order the variables are checked is:
# DEVSTACK_BRANCH then PROJECT_BRANCH
BRANCH_SPECIFIED=$(test -z "${DEVSTACK_BRANCH}${PROJECT_BRANCH}" || echo 'True')
if [[ "${BRANCH_SPECIFIED}" = "True" ]]; then
PROJ_BRANCH=$(get_project_branch DEVSTACK_BRANCH $PROJECT_BRANCH)
ENV_VARS="DEVSTACK_BRANCH' or 'PROJECT_BRANCH"
git_checkout "devstack" "$PATH_DEVSTACK_SRC" "$PROJ_BRANCH" "$ENV_VARS"
fi
exclaim "Installing devstack projects..."
# Ensures present user can get to the devstack dirs
sudo mkdir -p $PATH_DEVSTACK_OUTPUT
if [ ! -w $PATH_DEVSTACK_OUTPUT ]; then
sudo chown `whoami` $PATH_DEVSTACK_OUTPUT
fi
# Clones all of the code to where devstack expects it to be
pushd $PATH_DEVSTACK_OUTPUT
cmd_clone_projects do_not_force_update $REDSTACK_SCRIPTS/projects-list \
$REDSTACK_SCRIPTS/image-projects-list
popd
}
function install_reviews_on_top_of_devstack() {
exclaim "Putting gerrit review code on top of the existing devstack code"
run_review_for nova $PATH_NOVA $REVIEW_NOVA
run_review_for python-novaclient $PATH_PYTHON_NOVACLIENT $REVIEW_PYTHON_NOVACLIENT
run_review_for keystone $PATH_KEYSTONE $REVIEW_KEYSTONE
run_review_for python-keystoneclient $PATH_KEYSTONECLIENT $REVIEW_PYTHON_KEYSTONECLIENT
run_review_for python-openstackclient $PATH_OPENSTACKCLIENT $REVIEW_PYTHON_OPENSTACKCLIENT
run_review_for glance $PATH_GLANCE $REVIEW_GLANCE
run_review_for swift $PATH_SWIFT $REVIEW_SWIFT
run_review_for python-swiftclient $PATH_PYTHON_SWIFTCLIENT $REVIEW_PYTHON_SWIFTCLIENT
run_review_for trove $PATH_TROVE $REVIEW_TROVE
run_review_for python-troveclient $PATH_PYTHON_TROVECLIENT $REVIEW_PYTHON_TROVECLIENT
}
function run_review_for() {
# Splits based on colon in the REVIEW_ARG and pulls from
GIT_NAME=$1
PATH_ARG=$2
REVIEW_ARG=$3
for review in `echo $REVIEW_ARG| tr ":" "\n"`
do
# This should be the ref spec for what we pull
pushd $PATH_ARG
git_timed pull https://review.openstack.org/p/openstack/$GIT_NAME refs/changes/$review
popd
done
}
function fixup_broken_devstack() {
# Nothing to do here, devstack is working
:
}
# Delete all the lines from FILE_NAME between START_TAG and END_TAG
# Tags must appear at the beginning of a line
function clear_file_lines() {
local FILE_NAME=$1
local START_TAG=$2
local END_TAG=$3
sed -i "/^$START_TAG$/,/^$END_TAG$/{/^$START_TAG/!{/^$END_TAG/!d;}}" "$FILE_NAME"
}
# Checks to see if a variable with the same name as FILE_NAME exists.
# Returns 'true' if no varable exists or if the value of the variable
# is set to 'true' - returns the VAR_NAME to set otherwise.
# FILE_NAME is first converted to uppercase, the extension is removed
# and all remaining '.' and spaces are replaced with '_'
function check_filename_var() {
local FILE_NAME=$1
DEREF_VALUE=false
if [ -f "$FILE_NAME" ]; then
VAR_NAME=$(basename "$FILE_NAME" ".rc" | tr '[:lower:][:blank:][:punct:]' '[:upper:]__')
DEREF_VALUE=$(get_bool "$VAR_NAME" "true")
if [ "$DEREF_VALUE" != "true" ]; then
DEREF_VALUE=$VAR_NAME
fi
fi
echo "$DEREF_VALUE"
}
# Add the contents of one file to another, after the given tag
# Run through 'eval' if PARSE_FILE is true (defaults to true)
# Start with a blank line if BLANK_LINE_TO_START is true (defaults to false)
function add_file_contents() {
local FILE_NAME=$1
local FILE_TO_ADD=$2
local TAG=$3
local PARSE_FILE=${4:-true}
local BLANK_LINE_TO_START=${5:-false}
TEMP_FILE=".redstack.$$"
rm -f "$TEMP_FILE"
if [ "$BLANK_LINE_TO_START" = "true" ]; then
echo "" > "$TEMP_FILE"
fi
if [ -f "$FILE_TO_ADD" ]; then
echo "Adding $FILE_TO_ADD to $FILE_NAME"
echo "# Contents from $FILE_TO_ADD" >> "$TEMP_FILE"
if [ "$PARSE_FILE" = "true" ]; then
eval echo "\"$(cat "$FILE_TO_ADD")\"" >> "$TEMP_FILE"
else
cat "$FILE_TO_ADD" >> "$TEMP_FILE"
fi
echo "# End Of Contents from $FILE_TO_ADD" >> "$TEMP_FILE"
fi
echo "" >> "$TEMP_FILE"
sed -i "/^$TAG/r $TEMP_FILE" "$FILE_NAME"
rm -f "$TEMP_FILE"
}
function run_devstack() {
exclaim "Running devstack..."
# (Re)Creating this lock directory seems sure-fire.
rm -rf "$USERHOME/nova_locks"
mkdir -p "$USERHOME/nova_locks"
TROVE_PRESENT_TAG="# Trove-integration"
LOCAL_CONF_D=local.conf.d
CONF_MATCH="*.rc"
MARKER_TOKEN="#####"
USER_LOCAL_CONF=$(readlink -f "${USER_LOCAL_CONF:-$USERHOME/$USER_LOCAL_CONF_NAME}")
LOCALRC_OPTS_TAG="$MARKER_TOKEN Redstack Localrc Options $MARKER_TOKEN"
LOCALRC_OPTS_TAG_END="$MARKER_TOKEN End Of Redstack Localrc Options $MARKER_TOKEN"
USER_OPTS_TAG="$MARKER_TOKEN User Specified Options $MARKER_TOKEN"
USER_OPTS_TAG_END="$MARKER_TOKEN End Of User Specified Options $MARKER_TOKEN"
ADD_OPTS_TAG="$MARKER_TOKEN Additional Options $MARKER_TOKEN"
ADD_OPTS_TAG_END="$MARKER_TOKEN End Of Additional Options $MARKER_TOKEN"
set_trove_plugin_vars false
pushd "$PATH_DEVSTACK_SRC"
DEVSTACK_LOCAL_CONF=$LOCAL_CONF
# remain backwards compatible with existing localrc files
if [ -f "$LOCALRC" ]; then
DEVSTACK_LOCAL_CONF=$LOCALRC
echo "Old-style devstack config file $PATH_DEVSTACK_SRC/$DEVSTACK_LOCAL_CONF found."
echo "Consider removing to generate the preferred-sytle config file $LOCAL_CONF."
fi
if [ -f "$DEVSTACK_LOCAL_CONF" ]; then
# Check if we have already configured the devstack config file
already_in_conf=$(grep "$TROVE_PRESENT_TAG" "$DEVSTACK_LOCAL_CONF" | wc -l)
if [ "$already_in_conf" == 0 ]; then
# We can no longer append to an existing old-style localrc file
if [ "$DEVSTACK_LOCAL_CONF" == "$LOCALRC" ]; then
echo "The devstack config file $PATH_DEVSTACK_SRC/$DEVSTACK_LOCAL_CONF is too old to append to."
echo "Please remove and try again."
exit 1
fi
# Otherwise append the redstack version to the existing file
eval echo "\"$(cat "$REDSTACK_SCRIPTS/$DEFAULT_LOCALRC")\"" >> "$DEVSTACK_LOCAL_CONF"
fi
else
# If a devstack config file doesn't exist, create it
eval echo "\"$(cat "$REDSTACK_SCRIPTS/$DEFAULT_LOCAL_CONF")\"" > "$DEVSTACK_LOCAL_CONF"
fi
# We can only replace sections from the LOCAL_CONF style files
if [ "$DEVSTACK_LOCAL_CONF" == "$LOCAL_CONF" ]; then
# Clear out all the options
clear_file_lines "$DEVSTACK_LOCAL_CONF" "$LOCALRC_OPTS_TAG" "$LOCALRC_OPTS_TAG_END"
clear_file_lines "$DEVSTACK_LOCAL_CONF" "$USER_OPTS_TAG" "$USER_OPTS_TAG_END"
clear_file_lines "$DEVSTACK_LOCAL_CONF" "$ADD_OPTS_TAG" "$ADD_OPTS_TAG_END"
# Add the main localrc file
PARSE_FILE="true"
BLANK_LINE_TO_START="true"
if [ -f "$REDSTACK_SCRIPTS/$DEFAULT_LOCALRC" ]; then
add_file_contents "$DEVSTACK_LOCAL_CONF" "$REDSTACK_SCRIPTS/$DEFAULT_LOCALRC" "$LOCALRC_OPTS_TAG" "$PARSE_FILE" "$BLANK_LINE_TO_START"
fi
# Add any user options
PARSE_FILE="false"
BLANK_LINE_TO_START="true"
if [ -f "$USER_LOCAL_CONF" ]; then
add_file_contents "$DEVSTACK_LOCAL_CONF" "$USER_LOCAL_CONF" "$USER_OPTS_TAG" "$PARSE_FILE" "$BLANK_LINE_TO_START"
fi
# Add all the files in the LOCAL_CONF_D directory that match CONF_MATCH (except for sample files)
# and that aren't excluded. Files are excluded by having a variable
# 'FILENAME_IN_UPPERCASE_MINUS_RC=false' in redstack.rc
# For Example: USING_VAGRANT=false (for the using_vagrant.rc file).
PARSE_FILE="true"
BLANK_LINE_TO_START="false"
while IFS= read -r -d '' CONF_FILE
do
FILE_NAME_VAR=$(check_filename_var "$CONF_FILE")
if [ "$FILE_NAME_VAR" = "true" ]; then
add_file_contents "$DEVSTACK_LOCAL_CONF" "$CONF_FILE" "$ADD_OPTS_TAG" "$PARSE_FILE" "$BLANK_LINE_TO_START"
else
echo "Skipping $CONF_FILE"
echo "Use $FILE_NAME_VAR=true to include"
fi
done < <(find "$REDSTACK_SCRIPTS/${LOCAL_CONF_D}" -name "${CONF_MATCH}" -follow -not -name "sample*.rc" -type f -print0)
# this is to add a blank line for readability
add_file_contents "$DEVSTACK_LOCAL_CONF" "" "$ADD_OPTS_TAG"
fi
./stack.sh
popd
}
function cmd_install() {
sudo mkdir -p $TROVE_LOGDIR # Creates TROVE_LOGDIR if it does not exist
if [ ! -w $TROVE_LOGDIR ]; then
sudo chown `whoami` $TROVE_LOGDIR
fi
install_prep_packages
install_devstack_code
install_reviews_on_top_of_devstack
fixup_broken_devstack
run_devstack
exclaim "${COLOR_GREEN}FINISHED INSTALL${COLOR_NONE}"
}
###############################################################################
# Build the image
# see functions_qemu
###############################################################################
# Grab a numbered field from python prettytable output
# Fields are numbered starting with 1
# Reverse syntax is supported: -1 is the last field, -2 is second to last, etc.
# get_field field-number
function get_field() {
while read data; do
if [ "$1" -lt 0 ]; then
field="(\$(NF$1))"
else
field="\$$(($1 + 1))"
fi
echo "$data" | awk -F'[ \t]*\\|[ \t]*' "{print $field}"
done
}
function get_glance_id () {
echo `$@ | grep ' id ' | get_field 2`
}
function set_bin_path() {
if is_fedora; then
sed -i "s|%bin_path%|/usr/bin|g" $TEST_CONF
else
sed -i "s|%bin_path%|/usr/local/bin|g" $TEST_CONF
fi
}
function set_mysql_pkg() {
if is_fedora; then
MYSQL_PKG="mysql-community-server"
else
MYSQL_PKG="mysql-server-5.6"
fi
}
function cmd_set_datastore() {
IMAGEID=$1
DATASTORE_TYPE=$2
# rd_manage datastore_update <datastore_name> <default_version>
rd_manage datastore_update "$DATASTORE_TYPE" ""
PACKAGES=${PACKAGES:-""}
if [ "$DATASTORE_TYPE" == "mysql" ]; then
set_mysql_pkg
PACKAGES=${PACKAGES:-$MYSQL_PKG}
VERSION="5.6"
elif [ "$DATASTORE_TYPE" == "percona" ]; then
PACKAGES=${PACKAGES:-"percona-server-server-5.6"}
VERSION="5.6"
elif [ "$DATASTORE_TYPE" == "pxc" ]; then
PACKAGES=${PACKAGES:-"percona-xtradb-cluster-server-5.6"}
VERSION="5.6"
elif [ "$DATASTORE_TYPE" == "mariadb" ]; then
PACKAGES=${PACKAGES:-"mariadb-server"}
VERSION="10.1"
elif [ "$DATASTORE_TYPE" == "mongodb" ]; then
PACKAGES=${PACKAGES:-"mongodb-org"}
VERSION="3.0"
elif [ "$DATASTORE_TYPE" == "redis" ]; then
PACKAGES=${PACKAGES:-"redis-server"}
VERSION="3.0"
elif [ "$DATASTORE_TYPE" == "cassandra" ]; then
PACKAGES=${PACKAGES:-"cassandra"}
VERSION="2.1.0"
elif [ "$DATASTORE_TYPE" == "couchbase" ]; then
PACKAGES=${PACKAGES:-"couchbase-server"}
VERSION="2.2.0"
elif [ "$DATASTORE_TYPE" == "postgresql" ]; then
PACKAGES=${PACKAGES:-"postgresql-9.4"}
VERSION="9.4"
elif [ "$DATASTORE_TYPE" == "couchdb" ]; then
PACKAGES=${PACKAGES:-"couchdb"}
VERSION="1.6.1"
elif [ "$DATASTORE_TYPE" == "vertica" ]; then
PACKAGES=${PACKAGES:-"vertica"}
VERSION="7.1"
elif [ "$DATASTORE_TYPE" == "db2" ]; then
PACKAGES=${PACKAGES:-""}
VERSION="10.5"
else
echo "Unrecognized datastore type. ($DATASTORE_TYPE)"
exit 1
fi
sed -i "s/%datastore_type%/$DATASTORE_TYPE/g" $TEST_CONF
sed -i "s/%datastore_version%/$VERSION/g" $TEST_CONF
#rd_manage datastore_version_update <datastore_name> <version_name> <datastore_manager> <image_id> <packages> <active>
rd_manage datastore_version_update "$DATASTORE_TYPE" "$VERSION" "$DATASTORE_TYPE" $IMAGEID "$PACKAGES" 1
rd_manage datastore_version_update "$DATASTORE_TYPE" "inactive_version" "manager1" $IMAGEID "" 0
rd_manage datastore_update "$DATASTORE_TYPE" "$VERSION"
rd_manage datastore_update Test_Datastore_1 ""
if [ -f "$PATH_TROVE"/trove/templates/$DATASTORE_TYPE/validation-rules.json ]; then
# add the configuration parameters to the database for the kick-start datastore
rd_manage db_load_datastore_config_parameters "$DATASTORE_TYPE" "$VERSION" "$PATH_TROVE"/trove/templates/$DATASTORE_TYPE/validation-rules.json
fi
cmd_stop
iniset $TROVE_CONF DEFAULT default_datastore "$DATASTORE_TYPE"
sleep 1.5
cmd_start
}
###############################################################################
# Run Unit Tests
###############################################################################
function cmd_unit_tests() {
exclaim "Running Trove Unit Tests..."
$PATH_TROVE/run_tests.sh -N
}
###############################################################################
# Start various OpenStack daemons interactively in a screen session
###############################################################################
function cmd_start_deps() {
if ! sudo vgs $VOLUME_GROUP; then
exclaim "Reconnecting Volume Group to Backing File"
sudo losetup -f --show ${VOLUME_BACKING_FILE}
fi
if ! egrep -q ${SWIFT_DATA_DIR}/drives/sdb1 /proc/mounts; then
exclaim "Re-mounting Swift Disk Image"
sudo mount -t xfs -o loop,noatime,nodiratime,nobarrier,logbufs=8 ${SWIFT_DISK_IMAGE} ${SWIFT_DATA_DIR}/drives/sdb1 || true
fi
if [[ -e $PATH_DEVSTACK_SRC/stack-screenrc ]]; then
screen -dmS stack -c $PATH_DEVSTACK_SRC/stack-screenrc
fi
}
function cmd_stop_deps() {
if screen -ls | grep -q stack; then
screen -S stack -X quit
rm -f $DEST/status/stack/*
fi
}
###############################################################################
# Initialize Trove
###############################################################################
function rd_manage() {
pushd $PATH_TROVE
$TROVE_BIN_DIR/trove-manage --config-file=$TROVE_CONF "$@"
popd
}
function install_test_packages() {
sudo -H $HTTP_PROXY pip install openstack.nose_plugin proboscis pexpect
}
function mod_confs() {
DATASTORE_TYPE=$1
sudo install -b --mode 0664 $REDSTACK_SCRIPTS/conf/test_begin.conf $TEST_CONF
TROVE_REPORT_DIR=$REDSTACK_SCRIPTS/../report/
EXTRA_CONF=$REDSTACK_SCRIPTS/conf/test.extra.conf
if [ -e $EXTRA_CONF ]; then
cat $EXTRA_CONF >> $TEST_CONF
fi
# Append datastore specific configuration file
DATASTORE_CONF=$REDSTACK_SCRIPTS/conf/$DATASTORE_TYPE.conf
if [ ! -f $DATASTORE_CONF ]; then
exclaim "Datastore configuration file ${DATASTORE_CONF} not found"
exit 1
fi
cat $DATASTORE_CONF | sudo tee -a $TEST_CONF > /dev/null
cat $REDSTACK_SCRIPTS/conf/test_end.conf | sudo tee -a $TEST_CONF > /dev/null
#Add the paths to the test conf
sed -i "s,%report_directory%,$TROVE_REPORT_DIR,g" $TEST_CONF
sed -i "s,%keystone_path%,$PATH_KEYSTONE,g" $TEST_CONF
sed -i "s,%nova_path%,$PATH_NOVA,g" $TEST_CONF
sed -i "s,%glance_path%,$PATH_GLANCE,g" $TEST_CONF
sed -i "s,%trove_path%,$PATH_TROVE,g" $TEST_CONF
sed -i "s,%service_host%,$SERVICE_HOST,g" $TEST_CONF
sed -i "s,%swifth_path%,$PATH_SWIFT,g" $TEST_CONF
# Add the region name into test.conf
sed -i "s/%region_name%/${REGION_NAME}/g" $TEST_CONF
# Add the tenant id's into test.conf
sed -i "s/%admin_tenant_id%/$(get_attribute_id project admin 1)/g" $TEST_CONF
sed -i "s/%alt_demo_tenant_id%/$(get_attribute_id project alt_demo 1)/g" $TEST_CONF
sed -i "s/%demo_tenant_id%/$(get_attribute_id project demo 1)/g" $TEST_CONF
sed -i "s/%admin_password%/$ADMIN_PASSWORD/g" $TEST_CONF
# Enable neutron tests if needed
sed -i "s/%neutron_enabled%/$ENABLE_NEUTRON/g" $TEST_CONF
# If neutron is enabled, we create a shared network and write this info to the
# confs so that the integration tests can use it.
if [[ $ENABLE_NEUTRON = true ]]; then
management_network_id=$(neutron --os-username=admin --os-password=$ADMIN_PASSWORD --os-tenant-name=admin --os-auth-url=$TROVE_AUTH_ENDPOINT net-list | awk '/ alt-private / {print $2}')
management_subnet=$(neutron --os-username=admin --os-password=$ADMIN_PASSWORD --os-tenant-name=admin --os-auth-url=$TROVE_AUTH_ENDPOINT subnet-list | awk '/ alt-private-subnet / {print $2}')
echo "Using neutron network $management_network_id and subnet $management_subnet"
sed -i "s,%shared_network%,$management_network_id,g" $TEST_CONF
sed -i "s,%shared_network_subnet%,$management_subnet,g" $TEST_CONF
else
# do not leave invalid keys in the configuration when using Nova for networking
sed -i "/%shared_network%/d" $TEST_CONF
sed -i "/%shared_network_subnet%/d" $TEST_CONF
fi
if [ "$DATASTORE_TYPE" = "vertica" ]; then
# Vertica needs more time than mysql for its boot/start/stop operations.
setup_cluster_configs cluster_member_count 3
elif [ "$DATASTORE_TYPE" = "pxc" ]; then
setup_cluster_configs min_cluster_member_count 2
elif [ "$DATASTORE_TYPE" = "cassandra" ]; then
setup_cluster_configs cluster_member_count 2
elif [ "$DATASTORE_TYPE" = "mongodb" ]; then
setup_cluster_configs cluster_member_count 2
# Decrease the number of required config servers per cluster to save resources.
iniset $TROVE_CONF $DATASTORE_TYPE num_config_servers_per_cluster 1
fi
set_bin_path
}
function setup_cluster_configs() {
# Setting cluster_member_count to 2 to decrease cluster spawn time.
iniset $TROVE_CONF $DATASTORE_TYPE $1 $2
}
# Add useful flavors for testing (with corresponding *.resize flavors)
function add_test_flavors() {
# name id ram root_vol vcpu
# the ram and vcpu for name.resize are automatically calculated
# eph and non-eph flavors are created for each entry
add_flavor 'tiny' 10 512 3 1
add_flavor 'small' 15 768 3 1
add_flavor 'small' 16 768 4 1
add_flavor 'small' 17 768 5 1
add_flavor 'medium' 20 1024 4 1
add_flavor 'medium' 21 1024 5 1
add_flavor 'large' 25 2048 5 1
add_flavor 'large' 26 2048 10 1
add_flavor 'large' 27 2048 15 1
# This will allow Nova to create an instance, but not enough disk to boot the image
add_flavor 'fault_1' 30 512 1 1
# This should be enough memory to cause Nova to fail entirely due to too much allocation
add_flavor 'fault_2' 31 131072 5 1
}
function cmd_test_init() {
exclaim 'Initializing Configuration for Running Tests...'
exclaim "Installing python test packages."
install_test_packages
exclaim "Modifying test.conf and guest.conf with appropriate values."
mod_confs $1
exclaim "Creating Test Flavors."
add_test_flavors
if [[ -n $KEY_DIR ]]; then
exclaim "Installing the SSH key from $KEY_DIR to the test environment."
mkdir -m 700 -p $USERHOME/.ssh
install -b --mode 0400 $KEY_DIR/id_rsa $USERHOME/.ssh
cat $KEY_DIR/authorized_keys >> $USERHOME/.ssh/authorized_keys
chmod 600 $USERHOME/.ssh/authorized_keys
fi
}
function cmd_build_image() {
IMAGE_DATASTORE_TYPE=${1:-'mysql'}
ESCAPED_PATH_TROVE=${2:-'\/opt\/stack\/trove'}
HOST_SCP_USERNAME=${3:-'ubuntu'}
GUEST_USERNAME=${4:-'ubuntu'}
exclaim "Ensuring we have all packages needed to build image."
sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS update
sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS install qemu
sudo -H $HTTP_PROXY pip install --upgrade pip dib-utils
install_devstack_code
cmd_clone_projects do_not_force_update $REDSTACK_SCRIPTS/image-projects-list
exclaim "Use tripleo-diskimagebuilder to actually build the Trove Guest Agent Image."
build_guest_image $IMAGE_DATASTORE_TYPE
}
function cmd_build_and_upload_image() {
local IMAGE_URL=""
# Use /tmp as file_cache
FILES=/tmp
if [[ -n $IMAGE_DOWNLOAD_URL ]]; then
exclaim "Downloading and using cached image"
IMAGE_URL=$IMAGE_DOWNLOAD_URL
else
exclaim "Trying to build image"
build_guest_image $1
QCOW_IMAGE=`find $VM_PATH -name '*.qcow2'`
IMAGE_URL="file://$QCOW_IMAGE"
fi
# The devstack openrc has references to 'enable_plugin' which causes errors
# in the stable/juno and stable/kilo branches. These are safe to ignore.
set +e; source $PATH_DEVSTACK_SRC/openrc admin admin; set -e
TOKEN=$(openstack token issue | grep ' id ' | get_field 2)
GLANCE_IMAGEIDS=$(glance image-list | grep $(basename $IMAGE_URL .qcow2) | get_field 1)
if [[ -n $GLANCE_IMAGEIDS ]]; then
glance image-delete $GLANCE_IMAGEIDS
fi
GLANCE_IMAGEID=`get_glance_id upload_image $IMAGE_URL $TOKEN`
set +e; source $PATH_DEVSTACK_SRC/openrc demo demo; set -e
[[ -z "$GLANCE_IMAGEID" ]] && echo "Glance upload failed!" && exit 1
echo "IMAGE ID: $GLANCE_IMAGEID"
exclaim "Updating Datastores"
cmd_set_datastore $GLANCE_IMAGEID $1
}
function cmd_initialize() {
exclaim '(Re)Initializing Trove...'
pushd $PATH_DEVSTACK_SRC
./unstack.sh
./stack.sh
popd
}
###############################################################################
# Start Trove specific daemons interactively in a screen session
###############################################################################
function tr_screen_it {
if screen -ls | grep -q stack; then
echo "Starting $@..."
screen -S stack -p $1 -X stuff "$2"$'\015'
fi
}
function init_fake_mode() {
# Create a test conf which, unlike the conf which runs on a user's machine,
# takes advantage of the running keystone service we have in our VM.
# You could think of this fake mode, which runs in the VM as being
# slightly less fake than the default one which runs outside of it.
CONF_FILE=/tmp/trove.conf.test
cp $PATH_TROVE/etc/trove/trove.conf.test $CONF_FILE
# Switch keystone from the fake class to the real one.
sed -i \
"s/trove.tests.fakes.keystone/keystone.middleware.auth_token/g" \
$CONF_FILE
sed -i "s/log_file = rdtest.log//g" $CONF_FILE
sed -i "s/use_stderr = False/use_stderr = True/g" $CONF_FILE
cd $PATH_TROVE
set -e
rm -f trove_test.sqlite
set +e
$TROVE_BIN_DIR/trove-manage --config-file=$CONF_FILE db_sync
sqlite3 trove_test.sqlite \
"INSERT INTO datastores VALUES ('a00000a0-00a0-0a00-00a0-000a000000aa', \
'mysql', 'b00000b0-00b0-0b00-00b0-000b000000bb'); \
INSERT INTO datastores values ('e00000e0-00e0-0e00-00e0-000e000000ee', \
'Test_Datastore_1', ''); \
INSERT INTO datastore_versions VALUES ('b00000b0-00b0-0b00-00b0-000b000000bb', \
'a00000a0-00a0-0a00-00a0-000a000000aa', '5.6', \
'c00000c0-00c0-0c00-00c0-000c000000cc', $MYSQL_PKG, 1, 'mysql'); \
INSERT INTO datastore_versions VALUES ('d00000d0-00d0-0d00-00d0-000d000000dd', \
'a00000a0-00a0-0a00-00a0-000a000000aa', 'inactive_version', \
'', '', 0, 'manager1'); \
INSERT INTO datastore_configuration_parameters VALUES \
('00000000-0000-0000-0000-000000000001', \
'key_buffer_size', 'b00000b0-00b0-0b00-00b0-000b000000bb', \
0, 4294967296, 0, 'integer', 0, NULL); \
INSERT INTO datastore_configuration_parameters VALUES \
('00000000-0000-0000-0000-000000000002', \
'connect_timeout', 'b00000b0-00b0-0b00-00b0-000b000000bb', \
0, 65535, 1, 'integer', 0, NULL); \
INSERT INTO datastore_configuration_parameters VALUES \
('00000000-0000-0000-0000-000000000003', \
'join_buffer_size', 'b00000b0-00b0-0b00-00b0-000b000000bb', \
0, 4294967296, 0, 'integer', 0, NULL); \
INSERT INTO datastore_configuration_parameters VALUES \
('00000000-0000-0000-0000-000000000004', \
'local_infile', 'b00000b0-00b0-0b00-00b0-000b000000bb', \
0, 1, 0, 'integer', 0, NULL); \
INSERT INTO datastore_configuration_parameters VALUES \
('00000000-0000-0000-0000-000000000005', \
'collation_server', 'b00000b0-00b0-0b00-00b0-000b000000bb', \
0, NULL, NULL, 'string', 0, NULL); \
"
}
function cmd_start() {
if screen -ls | grep -q stack; then
set_trove_plugin_vars
USE_SCREEN=True
TOP_DIR=$PATH_DEVSTACK_SRC
LOGDIR=$TROVE_LOGDIR
RUNNING=$(screen -S stack -Q windows)
if [[ "$RUNNING" =~ " tr-" ]]; then
exclaim "${COLOR_RED}WARNING: Trove services appear to be running. Please run 'stop' or 'restart'${COLOR_NONE}"
else
source /dev/stdin < <(sed -n '/^function start_trove\(\)/,/^}/p' "$TROVE_SETUP_CMD_FILE")
start_trove
fi
else
echo "WARNING: Could not start Trove services because there is no stack screen session running"
fi
}
function cmd_start_fake() {
init_fake_mode
CONF_FILE=/tmp/trove.conf.test
tr_screen_it tr-fake "cd $PATH_TROVE; $TROVE_BIN_DIR/trove-fake-mode --config-file=$CONF_FILE $@"
}
function cmd_run() {
cd $PATH_TROVE; $TROVE_BIN_DIR/trove-api \
--config-file=$TROVE_CONF $@
}
function cmd_run_fake() {
init_fake_mode
CONF_FILE=/tmp/trove.conf.test
$TROVE_BIN_DIR/trove-fake-mode --config-file=$CONF_FILE $@
}
###############################################################################
# Stop any active Trove screen session
###############################################################################
function cmd_stop() {
if screen -ls | grep -q stack; then
set_trove_plugin_vars
rm -f $DEST/status/stack/tr-*
USE_SCREEN=True
source /dev/stdin < <(sed -n '/^function stop_trove\(\)/,/^}/p' "$TROVE_SETUP_CMD_FILE")
MAX_RETRY=5
COUNT=1
while true; do
RUNNING=$(screen -S stack -Q windows)
if [[ "$RUNNING" =~ " tr-" ]]; then
stop_trove
else
break
fi
((COUNT++))
if [ "$COUNT" -gt "$MAX_RETRY" ]; then
exclaim "${COLOR_RED}WARNING: Could not stop Trove services after ${MAX_RETRY} attempts${COLOR_NONE}"
break
fi
done
else
echo "WARNING: Could not stop Trove services because there is no stack screen session running"
fi
}
###############################################################################
# Run Integration Tests
###############################################################################
function cmd_int_tests() {
exclaim "Running Trove Integration Tests..."
if [ ! $USAGE_ENDPOINT ]; then
export USAGE_ENDPOINT=trove.tests.util.usage.FakeVerifier
fi
cd $REDSTACK_SCRIPTS
if [ $# -lt 1 ]; then
args="--group=blackbox"
else
args="$@"
fi
# -- verbose makes it prettier.
# -- logging-clear-handlers keeps the novaclient and other things from
# spewing logs to stdout.
args="$INT_TEST_OPTIONS -B $REDSTACK_TESTS/integration/int_tests.py --verbose --logging-clear-handlers $args"
echo "python $args"
python $args
}
function cmd_int_tests_simple() {
exclaim "Running Trove Simple Integration Tests..."
cd $REDSTACK_SCRIPTS
if [ $# -lt 1 ]; then
args="--group=simple_blackbox"
else
args="$@"
fi
# -- verbose makes it prettier.
# -- logging-clear-handlers keeps the novaclient and other things from
# spewing logs to stdout.
args="$INT_TEST_OPTIONS -B $REDSTACK_TESTS/integration/int_tests.py --verbose --logging-clear-handlers $args"
echo "python $args"
python $args
}
function cmd_int_tests_white_box() {
export PYTHONPATH=$PYTHONPATH:$PATH_TROVE
export PYTHONPATH=$PYTHONPATH:$PATH_NOVA
cmd_int_tests --test-config white_box=True \
--config-file=$TROVE_CONF \
--nova-flags=/etc/nova/nova.conf $@
}
function cmd_example_tests() {
set +e
cmd_stop
set -e
cmd_start_fake
sleep 3
echo "
{
\"directory\": \"$REDSTACK_TESTS/../apidocs/src/resources/samples/\",
\"auth_url\":\"http://$KEYSTONE_AUTH_HOST:35357/v2.0/tokens\",
\"api_url\":\"http://$SERVICE_HOST:8779\",
\"replace_host\":\"https://ord.databases.api.rackspacecloud.com\",
\"replace_dns_hostname\": \"e09ad9a3f73309469cf1f43d11e79549caf9acf2.rackspaceclouddb.com\",
\"username\":\"examples\",
\"password\":\"examples\",
\"tenant\":\"trove\"
}" > /tmp/example-tests.conf
python $REDSTACK_TESTS/examples/examples/example_generation.py /tmp/example-tests.conf
pushd $REDSTACK_TESTS/../apidocs
mvn clean
mvn generate-sources
popd
cmd_stop
}
###############################################################################
# Misc. tools
###############################################################################
function mysql_nova() {
echo mysql nova --execute "$@"
mysql -u root -p$MYSQL_PASSWORD nova --execute "$@"
}
function mysql_trove() {
echo mysql trove --execute "$@"
mysql -u root -p$MYSQL_PASSWORD trove --execute "$@"
}
function cmd_wipe_logs() {
for file in `ls $TROVE_LOGDIR/*.log`
do
echo "Reseting log file $file..."
echo "Reset at `date`" > $file
done
}
function cmd_rd_sql() {
mysql -u root -p$MYSQL_PASSWORD trove
}
function cmd_fake_sql() {
pushd $PATH_TROVE
sqlite3 trove_test.sqlite $@
popd
}
function cmd_vagrant_ssh() {
# Runs a command on a vagrant VM from the host machine.
VHOST=`vagrant ssh_config host | awk '/HostName/{print $2}'`
VUSER=`vagrant ssh_config host | awk '/User /{print $2}'`
VPORT=`vagrant ssh_config host | awk '/Port/{print $2}'`
VIDFILE=`vagrant ssh_config host | awk '/IdentityFile/{print $2}'`
echo ssh ${VUSER}@${VHOST} -p ${VPORT} -i ${VIDFILE} -o NoHostAuthenticationForLocalhost=yes "$@"
ssh ${VUSER}@${VHOST} -p ${VPORT} -i ${VIDFILE} -o NoHostAuthenticationForLocalhost=yes "$@"
}
function cmd_run_ci() {
exclaim "Running CI suite..."
set +e
cmd_stop_deps
cmd_stop
set -e
cmd_install
cmd_test_init $1
# The arg will be the image type
cmd_build_and_upload_image $1
# Test in fake mode.
exclaim "Testing in fake mode."
cmd_start_fake
FAKE_MODE=True cmd_int_tests
cmd_stop
# Test in real mode.
exclaim "Testing in real mode."
cmd_start
FAKE_MODE=False cmd_int_tests
}
function cmd_wipe_queues() {
# Obliterate rabbit.
for i in stop_app reset start_app "change_password guest $RABBIT_PASSWORD"; \
do sudo rabbitmqctl $i; done
}
function cmd_clear() {
cmd_int_tests --group=dbaas.api.instances.delete
clean_instances
mysql_nova "DELETE FROM instance_info_caches;"
mysql_nova "DELETE FROM instances;"
mysql_trove "DELETE FROM instances;"
mysql_trove "DELETE FROM service_statuses;"
cmd_wipe_queues
}
function exec_cmd_on_output() {
local output_cmd=$1
local exec_cmd=$2
local skip_id=${3:-""}
echo "Cleaning up objects from '${output_cmd}'"
exec $output_cmd|awk -vexec_cmd="$exec_cmd" -vskip_id="$skip_id" '{if (NR>3 && $2!="" && $2!=skip_id && $2!="|") { cmd = exec_cmd " " $2; print "Executing " cmd; system(cmd) } }'
}
function cmd_clean() {
echo "Cleaning up project '${OS_PROJECT_NAME}'"
# reset any stuck backups
mysql_trove "update backups set state='COMPLETED'"
# clear out any DS version metadata
mysql_trove "delete from datastore_version_metadata"
# reset any stuck instances, and clear all replicas
mysql_trove "update instances set task_id=2, slave_of_id=null"
# reset any stuck clusters
mysql_trove "update clusters set task_id=1"
# get rid of any extraneous quota usage
mysql_trove "delete from quota_usages"
source $PATH_DEVSTACK_SRC/openrc admin ${OS_PROJECT_NAME}
# delete any trove clusters
exec_cmd_on_output "trove cluster-list" "trove cluster-delete"
# delete any trove instances
exec_cmd_on_output "trove list" "trove delete"
# delete any backups
exec_cmd_on_output "trove backup-list" "trove backup-delete"
# sleep for a bit and then clean up
# any remaining nova instances or cinder volumes
sleep 5
exec_cmd_on_output "nova list" "nova delete"
sleep 2
exec_cmd_on_output "cinder list" "cinder delete"
# delete any config groups since all instances should be gone now
exec_cmd_on_output "trove configuration-list" "trove configuration-delete"
# delete any modules too
exec_cmd_on_output "trove module-list" "trove module-delete"
# make sure that security groups are also gone, except the default
exec_cmd_on_output "nova secgroup-list" "nova secgroup-delete" "1"
echo "If any errors occurred wait a few seconds and run the command again"
}
function cmd_kick_start() {
cmd_test_init $1
cmd_build_and_upload_image $1
}
function cmd_dsvm_gate_tests() {
DATASTORE_TYPE=${1:-'mysql'}
TEST_GROUP=${2:-${DATASTORE_TYPE}}
HOST_SCP_USERNAME=${3:-'jenkins'}
GUEST_USERNAME=${4:-'ubuntu'}
CONTROLLER_IP=${5:-'10.1.0.1'}
ESCAPED_PATH_TROVE=${6:-'\/opt\/stack\/new\/trove'}
if [[ $BRANCH_OVERRIDE == "stable/liberty" ]]; then
# Devstack in liberty doesn't copy the clouds.yaml file to /etc so we need to
# ensure we have access to the clouds.yaml file set up by devstack-gate
sudo mkdir -p ~/.config/openstack
sudo ln -s $DEST/.config/openstack/clouds.yaml ~/.config/openstack/clouds.yaml
sudo chown -R $(whoami) ~/.config
fi
# Devstack vm-gate runs as the jenkins user, but needs to connect to the guest image as ubuntu
echo "User=ubuntu" >> /home/jenkins/.ssh/config
# Fix iptables rules that prevent amqp connections from the devstack box to the guests
sudo iptables -D openstack-INPUT -j REJECT --reject-with icmp-host-prohibited || true
sudo chown -R $(whoami) /etc/trove
sudo chown -R $(whoami) $DEST/trove-integration
iniset $TROVE_GUESTAGENT_CONF DEFAULT rabbit_host $CONTROLLER_IP
iniset $TROVE_GUESTAGENT_CONF oslo_messaging_rabbit rabbit_hosts $CONTROLLER_IP
cd $DEST/trove-integration/scripts
sudo -H $HTTP_PROXY pip install --upgrade pip dib-utils
cmd_kick_start $DATASTORE_TYPE
# Update the local swift endpoint in the catalog to use the CONTROLLER_IP instead of 127.0.0.1
source $DEST/devstack/accrc/admin/admin
# NOTE(mriedem): We have to treat stable branches before liberty special
# due to constraints with older versions of python-openstackclient.
if [[ $BRANCH_OVERRIDE == "stable/juno" || $BRANCH_OVERRIDE == "stable/kilo" ]]; then
SWIFT_ENDPOINT=$(openstack endpoint list | grep 'swift' | get_field 1)
openstack endpoint create swift --region RegionOne --publicurl 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' \
--internalurl 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' --adminurl 'http://'$CONTROLLER_IP':8080'
openstack endpoint delete $SWIFT_ENDPOINT
else
OS_CLIENT_ARGS="--os-auth-type v3password --os-auth-url $KEYSTONE_AUTH_PROTOCOL://$KEYSTONE_AUTH_HOST:5000/v3 --os-identity-api-version=3"
SWIFT_ENDPOINTS=$(openstack endpoint list $OS_CLIENT_ARGS --service swift -c ID -f value)
openstack endpoint create $OS_CLIENT_ARGS swift public 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' --region RegionOne
openstack endpoint create $OS_CLIENT_ARGS swift internal 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' --region RegionOne
openstack endpoint create $OS_CLIENT_ARGS swift admin 'http://'$CONTROLLER_IP':8080' --region RegionOne
echo $SWIFT_ENDPOINTS | xargs -n 1 openstack endpoint delete $OS_CLIENT_ARGS
fi
cmd_int_tests --group=$TEST_GROUP
}
function cmd_reset_task() {
mysql_trove "UPDATE instances SET task_id=1 WHERE id='$1'"
}
function cmd_clone_projects() {
UPDATE_PROJECTS=$1
PROJECT_LIST_FILES=${@:2}
for project in $(cat $PROJECT_LIST_FILES); do
if [ ! -d $PATH_DEVSTACK_OUTPUT/$project ]; then
echo "Creating a new clone of $project..."
git_clone $GIT_OPENSTACK/"$project".git ${PATH_DEVSTACK_OUTPUT}/$project master
else
if [ $UPDATE_PROJECTS != "force_update" ]; then
echo "$project was already cloned or exists in a shared folder. Ignoring..."
else
echo "$project was already cloned. Pulling changes to update."
cd $PATH_DEVSTACK_OUTPUT/$project
git pull
fi
fi
# Switch to a branch if specified. The order the variables are checked is:
# <PROJECT>_BRANCH then PROJECT_CLIENT_BRANCH (if a client) then PROJECT_BRANCH
# Note: For the Trove project, only TROVE_BRANCH and PYTHON_TROVECLIENT_BRANCH are used
PROJECT_BRANCH_NAME=$(eval echo "${project}_BRANCH" | tr '[:lower:]-' '[:upper:]_')
PROJECT_BRANCH_VALUE=${!PROJECT_BRANCH_NAME}
# TROVE_BRANCH is defaulted to master if not set, so use the original value here
if [[ "$project" = "trove" ]]; then
PROJECT_BRANCH_VALUE=${TROVE_BRANCH_ORIG}
fi
BRANCH_SPECIFIED=$(test -z "${PROJECT_BRANCH_VALUE}${PROJECT_CLIENT_BRANCH}${PROJECT_BRANCH}" || echo 'True')
if [[ "${BRANCH_SPECIFIED}" = "True" ]]; then
# Set up the default branch and env var names for the project
DEFAULT_BRANCH="$PROJECT_BRANCH"
ENV_VARS="$PROJECT_BRANCH_NAME' or 'PROJECT_BRANCH"
# Don't use 'PROJECT_BRANCH' or 'PROJECT_CLIENT_BRANCH' for the Trove project
if [[ "$project" =~ "trove" ]]; then
DEFAULT_BRANCH=master
ENV_VARS="$PROJECT_BRANCH_NAME"
# Use 'PROJECT_CLIENT_BRANCH' first for clients
elif [[ "$project" =~ "client" ]]; then
DEFAULT_BRANCH="${PROJECT_CLIENT_BRANCH:-$PROJECT_BRANCH}"
ENV_VARS="$PROJECT_BRANCH_NAME' or 'PROJECT_CLIENT_BRANCH' or 'PROJECT_BRANCH"
fi
PROJ_BRANCH=$(get_project_branch $PROJECT_BRANCH_NAME $DEFAULT_BRANCH)
git_checkout "$project" "$PATH_DEVSTACK_OUTPUT/$project" "$PROJ_BRANCH" "$ENV_VARS"
fi
done
}
function cmd_repl() {
INT_TEST_OPTIONS=-i cmd_int_tests_white_box --repl --group=_does_not_exist_ $@
}
###############################################################################
# Process the user provided command and run the appropriate command
###############################################################################
# Let's not run this as the root user
if [ $EUID -eq 0 ]; then
echo "You are running this script as root. You need to run as a regular user"
exit 1
fi
# Set this to exit immediately on error
set -o errexit
# set_home_dir
set_http_proxy
set_trove_plugin_vars false
function print_usage() {
echo "Usage: $0 [command]"
echo "
Commands :
--setup environment--
install - Install all the required dependencies and bring up tr-api and tr-tmgr
- devstack config can be altered by using a USER_LOCAL_CONF file
which will be copied into devstack/local.conf on each 'install' run
(defaults to \$HOME/$USER_LOCAL_CONF_NAME)
- Set DEVSTACK_BRANCH to switch the branch/commit of devstack
(i.e. 'stable/kilo' or '7ef2462')
test-init - Configure the test configuration files and add keystone test users
build-image - Builds the vm image for the trove guest
initialize - Reinitialize the trove database, users, services, and test config
--helper for environment--
kick-start - kick start the setup of trove.
(redstack test-init/build-image in one step)
[mysql no-clean] no clean avoids rebuilding packages from scratch
- Set REBUILD_IMAGE=True to force rebuild (won't use cached image)
--trove dependency services--
start-deps - Start or resume daemons Trove depends on.
stop-deps - Kill daemons Trove depends on.
--trove services--
start - Start or resume daemons Trove depends on.
stop - Kill daemons Trove depends on.
restart - Runs stop then start for Trove services.
--tests--
unit-tests - Run the unit tests.dependencies
int-tests - Runs the integration tests (requires all daemons).
simple-tests - Runs the simple integration tests (requires all daemons).
dsvm-gate-tests - Configures and runs the int-tests in a devstack vm-gate environment.
--tools--
debug - Debug this script (shows all commands).
wipe-logs - Resets all log files.
rd-sql - Opens the Trove MySQL database.
vagrant-ssh - Runs a command from the host on the server.
clear - Destroy instances and rabbit queues.
clean - Clean up resources created by a failed test run.
run - Starts RD but not in a screen.
run-fake - Runs the server in fake mode.
update-projects - Git pull on all the daemons trove dependencies.
reset-task - Sets an instance task to NONE.
wipe-queues - Resets RabbitMQ queues.
"
exit 1
}
function run_command() {
# Print the available commands
if [ $# -lt 1 ]; then
print_usage
fi
case "$1" in
"install" ) cmd_install;;
"test-init" ) cmd_test_init $@;;
"build-image" ) shift; cmd_build_image $@;;
"initialize" ) cmd_initialize;;
"unit-tests" ) cmd_unit_tests;;
"start-deps" ) cmd_start_deps;;
"stop-deps" ) cmd_stop_deps;;
"start" ) cmd_start;;
"int-tests" ) shift; cmd_int_tests $@;;
"int-tests-wb" ) shift; cmd_int_tests_white_box $@;;
"simple-tests") shift; cmd_int_tests_simple $@;;
"stop" ) cmd_stop;;
"restart" ) cmd_stop; cmd_start;;
"wipe-logs" ) cmd_wipe_logs;;
"rd-sql" ) shift; cmd_rd_sql $@;;
"fake-sql" ) shift; cmd_fake_sql $@;;
"run-ci" ) shift; cmd_run_ci $@;;
"vagrant-ssh" ) shift; cmd_vagrant_ssh $@;;
"debug" ) shift; echo "Enabling debugging."; \
set -o xtrace; run_command $@;;
"clear" ) shift; cmd_clear $@;;
"clean" ) shift; cmd_clean $@;;
"run" ) shift; cmd_run $@;;
"kick-start" ) shift; cmd_kick_start $@;;
"dsvm-gate-tests" ) shift; cmd_dsvm_gate_tests $@;;
"run-fake" ) shift; cmd_run_fake $@;;
"start-fake" ) shift; cmd_start_fake $@;;
"update-projects" ) cmd_clone_projects force_update \
$REDSTACK_SCRIPTS/projects-list \
$REDSTACK_SCRIPTS/image-projects-list;;
"reset-task" ) shift; cmd_reset_task $@;;
"wipe-queues" ) shift; cmd_wipe_queues $@;;
"example-tests" ) shift; cmd_example_tests $@;;
"repl" ) shift; cmd_repl $@;;
"help" ) print_usage;;
* )
echo "'$1' not a valid command"
exit 1
esac
}
run_command $@
View Git Blame Copy Permalink