932 lines
34 KiB
Bash
Executable File
932 lines
34 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
###############################################################################
|
|
# RedStack, the Trove Dev Machine Controller #
|
|
###############################################################################
|
|
# #
|
|
# This script provides all the functionality to run all the steps from #
|
|
# setting up the environment, resetting the nova database to running the #
|
|
# test. #
|
|
# #
|
|
###############################################################################
|
|
|
|
REDSTACK_SCRIPTS=${REDSTACK_SCRIPTS:-`pwd`}
|
|
REDSTACK_TESTS=$REDSTACK_SCRIPTS/../tests/
|
|
|
|
USERHOME=$HOME
|
|
# Load options not checked into VCS.
|
|
if [ -f $REDSTACK_SCRIPTS/options.rc ]; then
|
|
. $REDSTACK_SCRIPTS/options.rc
|
|
fi
|
|
|
|
# Bail on errors.
|
|
set -e
|
|
|
|
# Get default host ip from interface
|
|
function get_default_host_ip() {
|
|
host_iface=$(ip route | sed -n '/^default/{ s/.*dev \(\w\+\)\s\+.*/\1/; p; }' | head -1)
|
|
echo `LC_ALL=C ip -f inet addr show ${host_iface} | awk '/inet/ {split($2,parts,"/"); print parts[1]}' | head -1`
|
|
}
|
|
|
|
# Load global configuration variables.
|
|
. $REDSTACK_SCRIPTS/redstack.rc
|
|
. $REDSTACK_SCRIPTS/reviews.rc
|
|
|
|
# allow overrides from devstack if already set
|
|
[[ -f $PATH_DEVSTACK_SRC/functions-common ]] && source $PATH_DEVSTACK_SRC/functions-common
|
|
[[ -f $PATH_DEVSTACK_SRC/functions ]] && source $PATH_DEVSTACK_SRC/functions
|
|
[[ -f $PATH_DEVSTACK_SRC/localrc ]] && source $PATH_DEVSTACK_SRC/localrc
|
|
|
|
# Public facing bits
|
|
[[ $ENABLE_NEUTRON = true ]] && BRIDGE_IP=172.24.4.1 || BRIDGE_IP=10.0.0.1
|
|
SERVICE_PROTOCOL=${SERVICE_PROTOCOL:-http}
|
|
SERVICE_TENANT=${SERVICE_TENANT:-service}
|
|
NETWORK_GATEWAY=${NETWORK_GATEWAY:-$BRIDGE_IP}
|
|
NETWORK_INTERFACE=${NETWORK_INTERFACE:-eth0}
|
|
NETWORK_SUBNET=${NETWORK_SUBNET:-10.0.0.0/24}
|
|
KEYSTONE_AUTH_HOST=${KEYSTONE_AUTH_HOST:-$SERVICE_HOST}
|
|
KEYSTONE_AUTH_PROTOCOL=${KEYSTONE_AUTH_PROTOCOL:-$SERVICE_PROTOCOL}
|
|
KEYSTONE_AUTH_PORT=${KEYSTONE_AUTH_PORT:-35357}
|
|
GLANCE_HOSTPORT=${GLANCE_HOSTPORT:-$SERVICE_HOST:9292}
|
|
TROVE_AUTH_ENDPOINT=$KEYSTONE_AUTH_PROTOCOL://$KEYSTONE_AUTH_HOST:$KEYSTONE_AUTH_PORT/v2.0
|
|
|
|
# PATH_TROVE more than likely has file separators, which sed does not like
|
|
# This will escape them
|
|
ESCAPED_PATH_TROVE=`echo $PATH_TROVE | sed 's/\//\\\\\//g'`
|
|
ESCAPED_REDSTACK_SCRIPTS=`echo $REDSTACK_SCRIPTS | sed 's/\//\\\\\//g'`
|
|
TROVE_AUTH_CACHE_DIR=${TROVE_AUTH_CACHE_DIR:-/var/cache/trove}
|
|
TROVE_LOGDIR=${TROVE_LOGDIR:-$REDSTACK_SCRIPTS/../report/logs}
|
|
|
|
# Load functions devstack style
|
|
. $REDSTACK_SCRIPTS/functions
|
|
. $REDSTACK_SCRIPTS/functions_qemu
|
|
|
|
# Support entry points installation of console scripts
|
|
if [[ -d $PATH_TROVE/bin ]]; then
|
|
TROVE_BIN_DIR=$PATH_TROVE/bin
|
|
else
|
|
TROVE_BIN_DIR=$(get_python_exec_prefix)
|
|
fi
|
|
|
|
# Allow sourcing config values from env.rc for overrides
|
|
if [ -f /tmp/integration/env.rc ]; then
|
|
source /tmp/integration/env.rc
|
|
fi
|
|
|
|
|
|
###############################################################################
|
|
# Utility functions
|
|
###############################################################################
|
|
|
|
APT_GET_ARGS="-y --allow-unauthenticated --force-yes"
|
|
|
|
function exclaim () {
|
|
echo "*******************************************************************************"
|
|
echo "$@"
|
|
echo "*******************************************************************************"
|
|
}
|
|
|
|
function pkg_install () {
|
|
echo Installing $@...
|
|
sudo -E DEBIAN_FRONTEND=noninteractive $HTTP_PROXY apt-get $APT_GET_ARGS install $@
|
|
}
|
|
|
|
function pkg_update () {
|
|
echo Updating $@...
|
|
sudo -E DEBIAN_FRONTEND=noninteractive $HTTP_PROXY apt-get $APT_GET_ARGS update $@
|
|
}
|
|
|
|
function set_home_dir() {
|
|
self="${0#./}"
|
|
base="${self%/*}"
|
|
current=`pwd`
|
|
if [ "$base" = "$self" ] ; then
|
|
HOME="$current/"
|
|
elif [[ $base =~ ^/ ]]; then
|
|
HOME="$base/"
|
|
else
|
|
HOME="$current/$base/"
|
|
fi
|
|
}
|
|
|
|
function set_http_proxy() {
|
|
if [ ! "${http_proxy}" = '' ]; then
|
|
HTTP_PROXY="http_proxy=$http_proxy https_proxy=$https_proxy"
|
|
fi
|
|
}
|
|
|
|
function get_ip_for_device() {
|
|
/sbin/ifconfig $1 | awk '/inet addr/{gsub(/addr:/,"");print $2}'
|
|
}
|
|
|
|
function ip_chunk() {
|
|
# Given 1-4 returns a bit of where the ip range starts.
|
|
# Full IP= `ip_chunk 1`.`ip_chunk 2`.`ip_chunk 3`.`ip_chunk 4`
|
|
get_ip_for_device $1 | cut -d. -f$2
|
|
}
|
|
|
|
function add_flavor() {
|
|
FLAVOR_NAME=$1
|
|
FLAVOR_ID=$2
|
|
FLAVOR_MEMORY_MB=$3
|
|
FLAVOR_ROOT_GB=$4
|
|
FLAVOR_VCPUS=$5
|
|
FLAVOR_EPHEMERAL=$6
|
|
|
|
if [[ -z $(nova --os-username=trove --os-password=$SERVICE_PASSWORD --os-tenant-name=$SERVICE_TENANT --os-auth-url=$TROVE_AUTH_ENDPOINT flavor-list | grep $FLAVOR_NAME) ]]; then
|
|
nova --os-username=trove --os-password=$SERVICE_PASSWORD --os-tenant-name=$SERVICE_TENANT --os-auth-url=$TROVE_AUTH_ENDPOINT flavor-create $FLAVOR_NAME $FLAVOR_ID $FLAVOR_MEMORY_MB $FLAVOR_ROOT_GB $FLAVOR_VCPUS --ephemeral $FLAVOR_EPHEMERAL
|
|
fi
|
|
}
|
|
|
|
function get_attribute_id() {
|
|
keystone --os-endpoint $TROVE_AUTH_ENDPOINT --os-token $SERVICE_TOKEN $1-list | grep " $2" | get_field $3
|
|
}
|
|
|
|
|
|
###############################################################################
|
|
# Install all the required dependencies
|
|
###############################################################################
|
|
|
|
function install_prep_packages() {
|
|
# Called before devstack
|
|
exclaim 'Updating dependencies (part 1a)...'
|
|
pkg_update
|
|
exclaim 'Installing dependencies (part 1b)...'
|
|
pkg_install git-core kvm-ipxe python-pip gettext
|
|
sudo $HTTP_PROXY pip install --upgrade pip dib-utils
|
|
}
|
|
|
|
function install_devstack_code() {
|
|
exclaim "Installing devstack..."
|
|
# Installs devstack (if needed).
|
|
if [ ! -d $PATH_DEVSTACK_SRC ]; then
|
|
echo "DevStack not in a shared folder, cloning from git."
|
|
mkdir -p $PATH_DEVSTACK_SRC
|
|
git clone $REPO_DEVSTACK $PATH_DEVSTACK_SRC
|
|
fi
|
|
|
|
exclaim "Installing devstack projects..."
|
|
# Ensures present user can get to the devstack dirs
|
|
sudo mkdir -p $PATH_DEVSTACK_OUTPUT
|
|
if [ ! -w $PATH_DEVSTACK_OUTPUT ]; then
|
|
sudo chown `whoami` $PATH_DEVSTACK_OUTPUT
|
|
fi
|
|
# Clones all of the code to where devstack expects it to be
|
|
pushd $PATH_DEVSTACK_OUTPUT
|
|
cmd_clone_projects do_not_force_update $REDSTACK_SCRIPTS/projects-list \
|
|
$REDSTACK_SCRIPTS/image-projects-list
|
|
popd
|
|
}
|
|
|
|
function install_reviews_on_top_of_devstack() {
|
|
exclaim "Putting gerrit review code on top of the existing devstack code"
|
|
run_review_for nova $PATH_NOVA $REVIEW_NOVA
|
|
run_review_for python-novaclient $PATH_PYTHON_NOVACLIENT $REVIEW_PYTHON_NOVACLIENT
|
|
run_review_for keystone $PATH_KEYSTONE $REVIEW_KEYSTONE
|
|
run_review_for python-keystoneclient $PATH_KEYSTONECLIENT $REVIEW_PYTHON_KEYSTONECLIENT
|
|
run_review_for glance $PATH_GLANCE $REVIEW_GLANCE
|
|
run_review_for swift $PATH_SWIFT $REVIEW_SWIFT
|
|
run_review_for python-swiftclient $PATH_PYTHON_SWIFTCLIENT $REVIEW_PYTHON_SWIFTCLIENT
|
|
run_review_for trove $PATH_TROVE $REVIEW_TROVE
|
|
run_review_for python-troveclient $PATH_PYTHON_TROVECLIENT $REVIEW_PYTHON_TROVECLIENT
|
|
}
|
|
|
|
function run_review_for() {
|
|
# Splits based on colon in the REVIEW_ARG and pulls from
|
|
GIT_NAME=$1
|
|
PATH_ARG=$2
|
|
REVIEW_ARG=$3
|
|
for review in `echo $REVIEW_ARG| tr ":" "\n"`
|
|
do
|
|
# This should be the ref spec for what we pull
|
|
pushd $PATH_ARG
|
|
git pull https://review.openstack.org/p/openstack/$GIT_NAME refs/changes/$review
|
|
popd
|
|
done
|
|
}
|
|
|
|
function fixup_broken_devstack() {
|
|
# Nothing to do here, devstack is working
|
|
:
|
|
}
|
|
|
|
function run_devstack() {
|
|
exclaim "Running devstack..."
|
|
|
|
# (Re)Creating this lock directory seems sure-fire.
|
|
rm -rf $USERHOME/nova_locks
|
|
mkdir -p $USERHOME/nova_locks
|
|
|
|
pushd $PATH_DEVSTACK_SRC
|
|
if [ -f localrc ]; then
|
|
#Check if we have already configured localrc
|
|
is_in_localrc=`grep "# Trove-integration" localrc | wc -l`
|
|
if [ $is_in_localrc == 0 ]; then
|
|
#Otherwise append them to the existing file
|
|
eval echo "\"$(cat $REDSTACK_SCRIPTS/localrc.rc)\"" >> localrc
|
|
fi
|
|
|
|
else
|
|
#If localrc doesn't exist, create it
|
|
eval echo "\"$(cat $REDSTACK_SCRIPTS/localrc.rc)\"" > localrc
|
|
fi
|
|
|
|
if [ -n "$USE_UUID_TOKEN" ]; then
|
|
echo "KEYSTONE_TOKEN_FORMAT=UUID" >> localrc
|
|
fi
|
|
|
|
if [ -n "$USING_VAGRANT" ]
|
|
then
|
|
eval echo "\"$(cat $REDSTACK_SCRIPTS/vagrant.rc)\"" >> localrc
|
|
fi
|
|
|
|
export MYSQL_HOST=$SERVICE_HOST
|
|
./stack.sh
|
|
popd
|
|
}
|
|
|
|
|
|
function cmd_install() {
|
|
mkdir -p $TROVE_LOGDIR # Creates TROVE_LOGDIR if it does not exist
|
|
install_prep_packages
|
|
install_devstack_code
|
|
install_reviews_on_top_of_devstack
|
|
fixup_broken_devstack
|
|
run_devstack
|
|
exclaim "FINISHED INSTALL"
|
|
}
|
|
|
|
|
|
###############################################################################
|
|
# Build the image
|
|
# see functions_qemu
|
|
###############################################################################
|
|
|
|
# Grab a numbered field from python prettytable output
|
|
# Fields are numbered starting with 1
|
|
# Reverse syntax is supported: -1 is the last field, -2 is second to last, etc.
|
|
# get_field field-number
|
|
function get_field() {
|
|
while read data; do
|
|
if [ "$1" -lt 0 ]; then
|
|
field="(\$(NF$1))"
|
|
else
|
|
field="\$$(($1 + 1))"
|
|
fi
|
|
echo "$data" | awk -F'[ \t]*\\|[ \t]*' "{print $field}"
|
|
done
|
|
}
|
|
|
|
function get_glance_id () {
|
|
echo `$@ | grep ' id ' | get_field 2`
|
|
}
|
|
|
|
function cmd_set_datastore() {
|
|
IMAGEID=$1
|
|
DATASTORE_TYPE=$2
|
|
# rd_manage datastore_update <datastore_name> <default_version>
|
|
rd_manage datastore_update "$DATASTORE_TYPE" ""
|
|
PACKAGES=${PACKAGES:-""}
|
|
if [ "$DATASTORE_TYPE" == "mysql" ]; then
|
|
PACKAGES=${PACKAGES:-"mysql-server-5.5"}
|
|
VERSION="5.5"
|
|
elif [ "$DATASTORE_TYPE" == "percona" ]; then
|
|
PACKAGES=${PACKAGES:-"percona-server-server-5.5"}
|
|
VERSION="5.5"
|
|
elif [ "$DATASTORE_TYPE" == "mongodb" ]; then
|
|
PACKAGES=${PACKAGES:-"mongodb"}
|
|
VERSION="2.0.4"
|
|
elif [ "$DATASTORE_TYPE" == "redis" ]; then
|
|
PACKAGES=${PACKAGES:-"redis-server"}
|
|
VERSION="2.8"
|
|
elif [ "$DATASTORE_TYPE" == "cassandra" ]; then
|
|
PACKAGES=${PACKAGES:-"cassandra"}
|
|
VERSION="2.0.6"
|
|
elif [ "$DATASTORE_TYPE" == "couchbase" ]; then
|
|
PACKAGES=${PACKAGES:-"couchbase-server"}
|
|
VERSION="2.2.0"
|
|
elif [ "$DATASTORE_TYPE" == "postgresql" ]; then
|
|
PACKAGES=${PACKAGES:-"postgresql-9.1"}
|
|
VERSION="9.1"
|
|
else
|
|
echo "Unrecognized datastore type. ($DATASTORE_TYPE)"
|
|
exit 1
|
|
fi
|
|
|
|
sed -i "s/%datastore_type%/$DATASTORE_TYPE/g" $TROVE_CONF_DIR/test.conf
|
|
sed -i "s/%datastore_version%/$VERSION/g" $TROVE_CONF_DIR/test.conf
|
|
|
|
#rd_manage datastore_version_update <datastore_name> <version_name> <datastore_manager> <image_id> <packages> <active>
|
|
rd_manage datastore_version_update "$DATASTORE_TYPE" "$VERSION" "$DATASTORE_TYPE" $IMAGEID "$PACKAGES" 1
|
|
rd_manage datastore_version_update "$DATASTORE_TYPE" "inactive_version" "manager1" $IMAGEID "" 0
|
|
rd_manage datastore_update "$DATASTORE_TYPE" "$VERSION"
|
|
rd_manage datastore_update Test_Datastore_1 ""
|
|
|
|
if [ -f "$PATH_TROVE"/trove/templates/$DATASTORE_TYPE/validation-rules.json ]; then
|
|
# add the configuration parameters to the database for the kick-start datastore
|
|
source $PATH_DEVSTACK_SRC/openrc admin admin
|
|
DATASTORE_VERSION_ID=`trove datastore-version-list "$DATASTORE_TYPE" | awk -v VERSION=$VERSION '{if ($4 == VERSION) { print $2 }}'`
|
|
rd_manage db_load_datastore_config_parameters "$DATASTORE_VERSION_ID" "$PATH_TROVE"/trove/templates/$DATASTORE_TYPE/validation-rules.json
|
|
fi
|
|
|
|
cmd_stop
|
|
iniset $TROVE_CONF_DIR/trove.conf DEFAULT default_datastore "$DATASTORE_TYPE"
|
|
sleep 1.5
|
|
cmd_start
|
|
}
|
|
|
|
###############################################################################
|
|
# Run Unit Tests
|
|
###############################################################################
|
|
|
|
function cmd_unit_tests() {
|
|
exclaim "Running Trove Unit Tests..."
|
|
$PATH_TROVE/run_tests.sh -N
|
|
}
|
|
|
|
###############################################################################
|
|
# Start various OpenStack daemons interactively in a screen session
|
|
###############################################################################
|
|
|
|
function cmd_start_deps() {
|
|
if ! sudo vgs $VOLUME_GROUP; then
|
|
exclaim "Reconnecting Volume Group to Backing File"
|
|
sudo losetup -f --show ${VOLUME_BACKING_FILE}
|
|
fi
|
|
|
|
if ! egrep -q ${SWIFT_DATA_DIR}/drives/sdb1 /proc/mounts; then
|
|
exclaim "Re-mounting Swift Disk Image"
|
|
sudo mount -t xfs -o loop,noatime,nodiratime,nobarrier,logbufs=8 ${SWIFT_DISK_IMAGE} ${SWIFT_DATA_DIR}/drives/sdb1
|
|
fi
|
|
|
|
if [[ -e $PATH_DEVSTACK_SRC/stack-screenrc ]]; then
|
|
screen -dmS stack -c $PATH_DEVSTACK_SRC/stack-screenrc
|
|
fi
|
|
}
|
|
|
|
function cmd_stop_deps() {
|
|
pushd $PATH_DEVSTACK_SRC
|
|
screen -S stack -X quit
|
|
popd
|
|
}
|
|
|
|
|
|
###############################################################################
|
|
# Initialize Trove
|
|
###############################################################################
|
|
|
|
function rd_manage() {
|
|
pushd $PATH_TROVE
|
|
$TROVE_BIN_DIR/trove-manage --config-file=$TROVE_CONF_DIR/trove.conf "$@"
|
|
popd
|
|
}
|
|
|
|
function install_test_packages() {
|
|
sudo $HTTP_PROXY pip install openstack.nose_plugin proboscis pexpect
|
|
}
|
|
|
|
function mod_confs() {
|
|
sudo install -b --mode 0664 $REDSTACK_SCRIPTS/conf/test_begin.conf $TROVE_CONF_DIR/test.conf
|
|
TROVE_REPORT_DIR=$REDSTACK_SCRIPTS/../report/
|
|
EXTRA_CONF=$REDSTACK_SCRIPTS/conf/test.extra.conf
|
|
if [ -e $EXTRA_CONF ]; then
|
|
cat $EXTRA_CONF >> $TROVE_CONF_DIR/test.conf
|
|
fi
|
|
cat $REDSTACK_SCRIPTS/conf/test_end.conf | sudo tee -a $TROVE_CONF_DIR/test.conf > /dev/null
|
|
|
|
#Add the paths to the test conf
|
|
sed -i "s,%report_directory%,$TROVE_REPORT_DIR,g" $TROVE_CONF_DIR/test.conf
|
|
sed -i "s,%keystone_path%,$PATH_KEYSTONE,g" $TROVE_CONF_DIR/test.conf
|
|
sed -i "s,%nova_path%,$PATH_NOVA,g" $TROVE_CONF_DIR/test.conf
|
|
sed -i "s,%glance_path%,$PATH_GLANCE,g" $TROVE_CONF_DIR/test.conf
|
|
sed -i "s,%trove_path%,$PATH_TROVE,g" $TROVE_CONF_DIR/test.conf
|
|
sed -i "s,%service_host%,$SERVICE_HOST,g" $TROVE_CONF_DIR/test.conf
|
|
sed -i "s,%swifth_path%,$PATH_SWIFT,g" $TROVE_CONF_DIR/test.conf
|
|
|
|
# Add the tenant id's into test.conf
|
|
sed -i "s/%admin_tenant_id%/$(get_attribute_id tenant admin 1)/g" $TROVE_CONF_DIR/test.conf
|
|
sed -i "s/%alt_demo_tenant_id%/$(get_attribute_id tenant alt_demo 1)/g" $TROVE_CONF_DIR/test.conf
|
|
sed -i "s/%demo_tenant_id%/$(get_attribute_id tenant demo 1)/g" $TROVE_CONF_DIR/test.conf
|
|
sed -i "s/%admin_password%/$ADMIN_PASSWORD/g" $TROVE_CONF_DIR/test.conf
|
|
|
|
# Enable neutron tests if needed
|
|
sed -i "s/%neutron_enabled%/$ENABLE_NEUTRON/g" $TROVE_CONF_DIR/test.conf
|
|
|
|
# If neutron is enabled, we create a shared network and write this info to the
|
|
# confs so that the integration tests can use it.
|
|
if [[ $ENABLE_NEUTRON = true ]]; then
|
|
# customer_network_id=$(neutron net-list | awk '/customer/ {print $2}')
|
|
# if [[ -z "$customer_network_id" ]]; then
|
|
# customer_network_id=$(neutron net-create customer | awk '/ id / {print $4}')
|
|
# customer_subnet_id=$(neutron subnet-create customer 10.0.1.0/24 --no-gateway | awk '/ id / {print $4}')
|
|
# fi
|
|
management_network_id=$(neutron net-list | awk '/private/ {print $2}')
|
|
management_subnet=$(neutron net-list | awk '/private/ {print $7}')
|
|
sed -i "s,%shared_network%,$management_network_id,g" $TROVE_CONF_DIR/test.conf
|
|
sed -i "s,%shared_network_subnet%,$management_subnet,g" $TROVE_CONF_DIR/test.conf
|
|
# sed -i "s,%private_network%,$customer_network_id,g" $TROVE_CONF_DIR/test.conf
|
|
# sed -i "s,%private_network_subnet%,10.0.1.0/24,g" $TROVE_CONF_DIR/test.conf
|
|
iniset $TROVE_CONF_DIR/trove.conf DEFAULT network_label_regex .*
|
|
iniset $TROVE_CONF_DIR/trove.conf DEFAULT ip_regex .*
|
|
iniset $TROVE_CONF_DIR/trove.conf DEFAULT black_list_regex ^10.0.1.*
|
|
iniset $TROVE_CONF_DIR/trove.conf DEFAULT default_neutron_networks $management_network_id
|
|
iniset $TROVE_CONF_DIR/trove.conf DEFAULT network_driver trove.network.neutron.NeutronDriver
|
|
iniset $TROVE_CONF_DIR/trove-taskmanager.conf DEFAULT network_driver trove.network.neutron.NeutronDriver
|
|
iniset $TROVE_CONF_DIR/trove-taskmanager.conf mysql tcp_ports 22,3306
|
|
fi
|
|
}
|
|
|
|
|
|
function add_test_flavors() {
|
|
add_flavor 'tinier' 6 506 10 1 0
|
|
add_flavor 'm1.rd-tiny' 7 512 3 1 0
|
|
add_flavor 'm1.rd-smaller' 8 768 3 1 0
|
|
add_flavor 'eph.rd-tiny' 9 512 3 1 1
|
|
add_flavor 'eph.rd-smaller' 10 768 3 1 2
|
|
}
|
|
|
|
function cmd_test_init() {
|
|
exclaim 'Initializing Configuration for Running Tests...'
|
|
|
|
exclaim "Installing python test packages."
|
|
install_test_packages
|
|
|
|
exclaim "Modifying test.conf and guest.conf with appropriate values."
|
|
mod_confs
|
|
|
|
exclaim "Creating Test Flavors."
|
|
add_test_flavors
|
|
|
|
if [[ -n $KEY_DIR ]]; then
|
|
exclaim "Installing the SSH key from $KEY_DIR to the test environment."
|
|
mkdir -p $USERHOME/.ssh
|
|
install -b --mode 0400 $KEY_DIR/id_rsa $USERHOME/.ssh
|
|
cat $KEY_DIR/authorized_keys >> $USERHOME/.ssh/authorized_keys
|
|
fi
|
|
}
|
|
|
|
function cmd_build_image() {
|
|
IMAGE_DATASTORE_TYPE=${1:-'mysql'}
|
|
ESCAPED_PATH_TROVE=${2:-'\/opt\/stack\/trove'}
|
|
HOST_SCP_USERNAME=${3:-'ubuntu'}
|
|
GUEST_USERNAME=${4:-'ubuntu'}
|
|
|
|
exclaim "Ensuring we have all packages needed to build image."
|
|
sudo $HTTP_PROXY apt-get -y update
|
|
sudo $HTTP_PROXY apt-get -y install qemu
|
|
cmd_clone_projects do_not_force_update $REDSTACK_SCRIPTS/image-projects-list
|
|
|
|
exclaim "Use tripleo-diskimagebuilder to actually build the Trove Guest Agent Image."
|
|
build_guest_image $IMAGE_DATASTORE_TYPE
|
|
}
|
|
|
|
function cmd_build_and_upload_image() {
|
|
local IMAGE_URL=""
|
|
# Use /tmp as file_cache
|
|
FILES=/tmp
|
|
if [[ -n $IMAGE_DOWNLOAD_URL ]]; then
|
|
exclaim "Downloading and using cached image"
|
|
IMAGE_URL=$IMAGE_DOWNLOAD_URL
|
|
else
|
|
exclaim "Trying to build image"
|
|
build_guest_image $1
|
|
QCOW_IMAGE=`find $VM_PATH -name '*.qcow2'`
|
|
IMAGE_URL="file://$QCOW_IMAGE"
|
|
fi
|
|
|
|
source $PATH_DEVSTACK_SRC/openrc admin admin
|
|
TOKEN=$(keystone token-get | grep ' id ' | get_field 2)
|
|
GLANCE_IMAGEIDS=$(glance image-list | grep $(basename $IMAGE_URL .qcow2) | get_field 1)
|
|
if [[ -n $GLANCE_IMAGEIDS ]]; then
|
|
glance image-delete $GLANCE_IMAGEIDS
|
|
fi
|
|
GLANCE_IMAGEID=`get_glance_id upload_image $IMAGE_URL $TOKEN`
|
|
source $PATH_DEVSTACK_SRC/openrc demo demo
|
|
[[ -z "$GLANCE_IMAGEID" ]] && echo "Glance upload failed!" && exit 1
|
|
echo "IMAGE ID: $GLANCE_IMAGEID"
|
|
|
|
exclaim "Updating Datastores"
|
|
cmd_set_datastore $GLANCE_IMAGEID $1
|
|
}
|
|
|
|
|
|
function cmd_initialize() {
|
|
exclaim '(Re)Initializing Trove...'
|
|
pushd $PATH_DEVSTACK_SRC
|
|
export MYSQL_HOST=$SERVICE_HOST
|
|
./unstack.sh
|
|
./stack.sh
|
|
popd
|
|
}
|
|
|
|
|
|
###############################################################################
|
|
# Start Trove specific daemons interactively in a screen session
|
|
###############################################################################
|
|
|
|
function tr_screen_it {
|
|
echo "Starting $@..."
|
|
screen -S stack -X screen -t $1 bash
|
|
screen -S stack -p $1 -X stuff "$2"$'\015'
|
|
}
|
|
|
|
function init_fake_mode() {
|
|
# Create a test conf which, unlike the conf which runs on a user's machine,
|
|
# takes advantage of the running keystone service we have in our VM.
|
|
# You could think of this fake mode, which runs in the VM as being
|
|
# slightly less fake than the default one which runs outside of it.
|
|
CONF_FILE=/tmp/trove.conf.test
|
|
cp $PATH_TROVE/etc/trove/trove.conf.test $CONF_FILE
|
|
# Switch keystone from the fake class to the real one.
|
|
sed -i \
|
|
"s/trove.tests.fakes.keystone/keystone.middleware.auth_token/g" \
|
|
$CONF_FILE
|
|
sed -i "s/log_file = rdtest.log//g" $CONF_FILE
|
|
sed -i "s/use_stderr = False/use_stderr = True/g" $CONF_FILE
|
|
cd $PATH_TROVE
|
|
set -e
|
|
rm -f trove_test.sqlite
|
|
set +e
|
|
$TROVE_BIN_DIR/trove-manage --config-file=$CONF_FILE db_sync
|
|
sqlite3 trove_test.sqlite \
|
|
"INSERT INTO datastores VALUES ('a00000a0-00a0-0a00-00a0-000a000000aa', \
|
|
'mysql', 'b00000b0-00b0-0b00-00b0-000b000000bb'); \
|
|
INSERT INTO datastores values ('e00000e0-00e0-0e00-00e0-000e000000ee', \
|
|
'Test_Datastore_1', ''); \
|
|
INSERT INTO datastore_versions VALUES ('b00000b0-00b0-0b00-00b0-000b000000bb', \
|
|
'a00000a0-00a0-0a00-00a0-000a000000aa', '5.5', \
|
|
'c00000c0-00c0-0c00-00c0-000c000000cc', 'mysql-server-5.5', 1, 'mysql'); \
|
|
INSERT INTO datastore_versions VALUES ('d00000d0-00d0-0d00-00d0-000d000000dd', \
|
|
'a00000a0-00a0-0a00-00a0-000a000000aa', 'inactive_version', \
|
|
'', '', 0, 'manager1'); \
|
|
INSERT INTO datastore_configuration_parameters VALUES \
|
|
('00000000-0000-0000-0000-000000000001', \
|
|
'key_buffer_size', 'b00000b0-00b0-0b00-00b0-000b000000bb', \
|
|
0, 4294967296, 0, 'integer', 0, NULL); \
|
|
INSERT INTO datastore_configuration_parameters VALUES \
|
|
('00000000-0000-0000-0000-000000000002', \
|
|
'connect_timeout', 'b00000b0-00b0-0b00-00b0-000b000000bb', \
|
|
0, 65535, 1, 'integer', 0, NULL); \
|
|
INSERT INTO datastore_configuration_parameters VALUES \
|
|
('00000000-0000-0000-0000-000000000003', \
|
|
'join_buffer_size', 'b00000b0-00b0-0b00-00b0-000b000000bb', \
|
|
0, 4294967296, 0, 'integer', 0, NULL); \
|
|
INSERT INTO datastore_configuration_parameters VALUES \
|
|
('00000000-0000-0000-0000-000000000004', \
|
|
'local_infile', 'b00000b0-00b0-0b00-00b0-000b000000bb', \
|
|
0, 1, 0, 'integer', 0, NULL); \
|
|
INSERT INTO datastore_configuration_parameters VALUES \
|
|
('00000000-0000-0000-0000-000000000005', \
|
|
'collation_server', 'b00000b0-00b0-0b00-00b0-000b000000bb', \
|
|
0, NULL, NULL, 'string', 0, NULL); \
|
|
"
|
|
}
|
|
|
|
function cmd_start() {
|
|
tr_screen_it tr-api "cd $PATH_TROVE; $TROVE_BIN_DIR/trove-api --config-file=$TROVE_CONF_DIR/trove.conf --debug 2>&1 | tee $TROVE_LOGDIR/trove-api.log"
|
|
tr_screen_it tr-tmgr "cd $PATH_TROVE; $TROVE_BIN_DIR/trove-taskmanager --config-file=$TROVE_CONF_DIR/trove-taskmanager.conf --debug 2>&1 | tee $TROVE_LOGDIR/trove-taskmanager.log"
|
|
tr_screen_it tr-cond "cd $PATH_TROVE; $TROVE_BIN_DIR/trove-conductor --config-file=$TROVE_CONF_DIR/trove-conductor.conf --debug 2>&1 | tee $TROVE_LOGDIR/trove-conductor.log"
|
|
}
|
|
|
|
function cmd_start_fake() {
|
|
init_fake_mode
|
|
CONF_FILE=/tmp/trove.conf.test
|
|
tr_screen_it tr-fake "cd $PATH_TROVE; $TROVE_BIN_DIR/trove-fake-mode --config-file=$CONF_FILE $@"
|
|
}
|
|
|
|
function cmd_run() {
|
|
cd $PATH_TROVE; $TROVE_BIN_DIR/trove-api \
|
|
--config-file=$TROVE_CONF_DIR/trove.conf $@
|
|
}
|
|
|
|
function cmd_run_fake() {
|
|
init_fake_mode
|
|
CONF_FILE=/tmp/trove.conf.test
|
|
$TROVE_BIN_DIR/trove-fake-mode --config-file=$CONF_FILE $@
|
|
}
|
|
|
|
###############################################################################
|
|
# Stop any active Trove screen session
|
|
###############################################################################
|
|
|
|
function cmd_stop() {
|
|
screen -S stack -p tr-api -X stuff $'\003 exit \015'
|
|
screen -S stack -p tr-tmgr -X stuff $'\003 exit \015'
|
|
screen -S stack -p tr-cond -X stuff $'\003 exit \015'
|
|
screen -S stack -p tr-fake -X stuff $'\003 exit \015'
|
|
}
|
|
|
|
|
|
###############################################################################
|
|
# Run Integration Tests
|
|
###############################################################################
|
|
|
|
function cmd_int_tests() {
|
|
exclaim "Running Trove Integration Tests..."
|
|
if [ ! $USAGE_ENDPOINT ]; then
|
|
export USAGE_ENDPOINT=trove.tests.util.usage.FakeVerifier
|
|
fi
|
|
cd $REDSTACK_SCRIPTS
|
|
export TEST_CONF=$TROVE_CONF_DIR/test.conf
|
|
if [ $# -lt 1 ]; then
|
|
args="--group=blackbox"
|
|
else
|
|
args="$@"
|
|
fi
|
|
|
|
# -- verbose makes it prettier.
|
|
# -- logging-clear-handlers keeps the novaclient and other things from
|
|
# spewing logs to stdout.
|
|
args="$INT_TEST_OPTIONS -B $REDSTACK_TESTS/integration/int_tests.py --verbose --logging-clear-handlers $args"
|
|
echo "python $args"
|
|
python $args
|
|
}
|
|
|
|
function cmd_int_tests_simple() {
|
|
exclaim "Running Trove Simple Integration Tests..."
|
|
cd $REDSTACK_SCRIPTS
|
|
export TEST_CONF=$TROVE_CONF_DIR/test.conf
|
|
if [ $# -lt 1 ]; then
|
|
args="--group=simple_blackbox"
|
|
else
|
|
args="$@"
|
|
fi
|
|
|
|
# -- verbose makes it prettier.
|
|
# -- logging-clear-handlers keeps the novaclient and other things from
|
|
# spewing logs to stdout.
|
|
args="$INT_TEST_OPTIONS -B $REDSTACK_TESTS/integration/int_tests.py --verbose --logging-clear-handlers $args"
|
|
echo "python $args"
|
|
python $args
|
|
}
|
|
|
|
function cmd_int_tests_white_box() {
|
|
export PYTHONPATH=$PYTHONPATH:$PATH_TROVE
|
|
export PYTHONPATH=$PYTHONPATH:$PATH_NOVA
|
|
cmd_int_tests --test-config white_box=True \
|
|
--config-file=$TROVE_CONF_DIR/trove.conf \
|
|
--nova-flags=/etc/nova/nova.conf $@
|
|
}
|
|
|
|
function cmd_example_tests() {
|
|
set +e
|
|
cmd_stop
|
|
set -e
|
|
cmd_start_fake
|
|
sleep 3
|
|
echo "
|
|
{
|
|
\"directory\": \"$REDSTACK_TESTS/../apidocs/src/resources/samples/\",
|
|
\"auth_url\":\"http://$KEYSTONE_AUTH_HOST:35357/v2.0/tokens\",
|
|
\"api_url\":\"http://$SERVICE_HOST:8779\",
|
|
\"replace_host\":\"https://ord.databases.api.rackspacecloud.com\",
|
|
\"replace_dns_hostname\": \"e09ad9a3f73309469cf1f43d11e79549caf9acf2.rackspaceclouddb.com\",
|
|
\"username\":\"examples\",
|
|
\"password\":\"examples\",
|
|
\"tenant\":\"trove\"
|
|
}" > /tmp/example-tests.conf
|
|
python $REDSTACK_TESTS/examples/examples/example_generation.py /tmp/example-tests.conf
|
|
pushd $REDSTACK_TESTS/../apidocs
|
|
mvn clean
|
|
mvn generate-sources
|
|
popd
|
|
cmd_stop
|
|
}
|
|
|
|
|
|
###############################################################################
|
|
# Misc. tools
|
|
###############################################################################
|
|
|
|
function mysql_nova() {
|
|
echo mysql nova --execute "$@"
|
|
mysql -u root -p$MYSQL_PASSWORD nova --execute "$@"
|
|
}
|
|
|
|
function mysql_trove() {
|
|
echo mysql trove --execute "$@"
|
|
mysql -u root -p$MYSQL_PASSWORD trove --execute "$@"
|
|
}
|
|
|
|
function cmd_wipe_logs() {
|
|
for file in `ls $TROVE_LOGDIR/*.log`
|
|
do
|
|
echo "Reseting log file $file..."
|
|
echo "Reset at `date`" > $file
|
|
done
|
|
}
|
|
|
|
function cmd_rd_sql() {
|
|
mysql -u root -p$MYSQL_PASSWORD trove
|
|
}
|
|
|
|
function cmd_fake_sql() {
|
|
pushd $PATH_TROVE
|
|
sqlite3 trove_test.sqlite $@
|
|
popd
|
|
}
|
|
|
|
function cmd_vagrant_ssh() {
|
|
# Runs a command on a vagrant VM from the host machine.
|
|
VHOST=`vagrant ssh_config host | awk '/HostName/{print $2}'`
|
|
VUSER=`vagrant ssh_config host | awk '/User /{print $2}'`
|
|
VPORT=`vagrant ssh_config host | awk '/Port/{print $2}'`
|
|
VIDFILE=`vagrant ssh_config host | awk '/IdentityFile/{print $2}'`
|
|
echo ssh ${VUSER}@${VHOST} -p ${VPORT} -i ${VIDFILE} -o NoHostAuthenticationForLocalhost=yes "$@"
|
|
ssh ${VUSER}@${VHOST} -p ${VPORT} -i ${VIDFILE} -o NoHostAuthenticationForLocalhost=yes "$@"
|
|
}
|
|
|
|
|
|
function cmd_run_ci() {
|
|
exclaim "Running CI suite..."
|
|
set +e
|
|
cmd_stop_deps
|
|
cmd_stop
|
|
set -e
|
|
cmd_install
|
|
cmd_test_init
|
|
# The arg will be the image type
|
|
cmd_build_and_upload_image $1
|
|
|
|
# Test in fake mode.
|
|
exclaim "Testing in fake mode."
|
|
cmd_start_fake
|
|
FAKE_MODE=True cmd_int_tests
|
|
cmd_stop
|
|
|
|
# Test in real mode.
|
|
exclaim "Testing in real mode."
|
|
cmd_start
|
|
FAKE_MODE=False cmd_int_tests
|
|
}
|
|
|
|
function cmd_wipe_queues() {
|
|
# Obliterate rabbit.
|
|
for i in stop_app reset start_app "change_password guest $RABBIT_PASSWORD"; \
|
|
do sudo rabbitmqctl $i; done
|
|
}
|
|
|
|
function cmd_clear() {
|
|
cmd_int_tests --group=dbaas.api.instances.delete
|
|
clean_instances
|
|
mysql_nova "DELETE FROM instance_info_caches;"
|
|
mysql_nova "DELETE FROM instances;"
|
|
mysql_trove "DELETE FROM instances;"
|
|
mysql_trove "DELETE FROM service_statuses;"
|
|
cmd_wipe_queues
|
|
}
|
|
|
|
function cmd_kick_start() {
|
|
cmd_test_init
|
|
cmd_build_and_upload_image $1
|
|
}
|
|
|
|
function cmd_dsvm_gate_tests() {
|
|
DATASTORE_TYPE=${1:-'mysql'}
|
|
HOST_SCP_USERNAME=${2:-'jenkins'}
|
|
GUEST_USERNAME=${3:-'ubuntu'}
|
|
NETWORK_GATEWAY=${4:-'10.1.0.1'}
|
|
ESCAPED_PATH_TROVE=${5:-'\/opt\/stack\/new\/trove'}
|
|
|
|
sudo chown -R $(whoami) /etc/trove
|
|
sudo chown -R $(whoami) $DEST/trove-integration
|
|
iniset $TROVE_CONF_DIR/trove-guestagent.conf DEFAULT rabbit_host $NETWORK_GATEWAY
|
|
cd $DEST/trove-integration/scripts
|
|
cmd_kick_start $DATASTORE_TYPE
|
|
cmd_int_tests --group=$DATASTORE_TYPE
|
|
}
|
|
|
|
function cmd_reset_task() {
|
|
mysql_trove "UPDATE instances SET task_id=1 WHERE id='$1'"
|
|
}
|
|
|
|
function cmd_clone_projects() {
|
|
UPDATE_PROJECTS=$1
|
|
PROJECT_LIST_FILES=${@:2}
|
|
for project in $(cat $PROJECT_LIST_FILES); do
|
|
if [ ! -d $PATH_DEVSTACK_OUTPUT/$project ]; then
|
|
echo "Creating a new clone of $project..."
|
|
git_clone $OPENSTACK_GITSOURCE/"$project".git ${PATH_DEVSTACK_OUTPUT}/$project master
|
|
else
|
|
if [ $UPDATE_PROJECTS != "force_update" ]; then
|
|
echo "$project was already cloned or exists in a shared folder. Ignoring..."
|
|
else
|
|
echo "$project was already cloned. Pulling changes to update."
|
|
cd $PATH_DEVSTACK_OUTPUT/$project
|
|
git pull
|
|
fi
|
|
fi
|
|
done
|
|
}
|
|
|
|
function cmd_repl() {
|
|
INT_TEST_OPTIONS=-i cmd_int_tests_white_box --repl --group=_does_not_exist_ $@
|
|
}
|
|
|
|
|
|
###############################################################################
|
|
# Process the user provided command and run the appropriate command
|
|
###############################################################################
|
|
|
|
# Let's not run this as the root user
|
|
if [ $EUID -eq 0 ]; then
|
|
echo "You are running this script as root. You need to run as a regular user"
|
|
exit 1
|
|
fi
|
|
|
|
# Set this to exit immediately on error
|
|
set -o errexit
|
|
|
|
set_home_dir
|
|
set_http_proxy
|
|
|
|
function print_usage() {
|
|
echo "Usage: $0 [command]"
|
|
echo "
|
|
Commands :
|
|
--setup environment--
|
|
install - Install all the required dependencies and bring up tr-api and tr-tmgr
|
|
test-init - Configure the test configuration files and add keystone test users
|
|
build-image - Builds the vm image for the trove guest
|
|
initialize - Reinitialize the trove database, users, services, and test config
|
|
|
|
--helper for environment--
|
|
kick-start - kick start the setup of trove.
|
|
(redstack test-init/build-image in one step)
|
|
[mysql no-clean] no clean avoids rebuilding packages from scratch
|
|
|
|
--trove dependency services--
|
|
start-deps - Start or resume daemons Trove depends on.
|
|
stop-deps - Kill daemons Trove depends on.
|
|
|
|
--trove services--
|
|
start - Start or resume daemons Trove depends on.
|
|
stop - Kill daemons Trove depends on.
|
|
|
|
--tests--
|
|
unit-tests - Run the unit tests.dependencies
|
|
int-tests - Runs the integration tests (requires all daemons).
|
|
simple-tests - Runs the simple integration tests (requires all daemons).
|
|
dsvm-gate-tests - Configures and runs the int-tests in a devstack vm-gate environment.
|
|
|
|
--tools--
|
|
debug - Debug this script (shows all commands).
|
|
wipe-logs - Resets all log files.
|
|
rd-sql - Opens the Trove MySQL database.
|
|
vagrant-ssh - Runs a command from the host on the server.
|
|
clear - Destroy instances and rabbit queues.
|
|
run - Starts RD but not in a screen.
|
|
run-fake - Runs the server in fake mode.
|
|
update-projects - Git pull on all the daemons trove dependencies.
|
|
reset-task - Sets an instance task to NONE.
|
|
wipe-queues - Resets RabbitMQ queues.
|
|
"
|
|
exit 1
|
|
}
|
|
|
|
function run_command() {
|
|
# Print the available commands
|
|
if [ $# -lt 1 ]; then
|
|
print_usage
|
|
fi
|
|
|
|
case "$1" in
|
|
"install" ) cmd_install;;
|
|
"test-init" ) cmd_test_init;;
|
|
"build-image" ) shift; cmd_build_image $@;;
|
|
"initialize" ) cmd_initialize;;
|
|
"unit-tests" ) cmd_unit_tests;;
|
|
"start-deps" ) cmd_start_deps;;
|
|
"stop-deps" ) cmd_stop_deps;;
|
|
"start" ) cmd_start;;
|
|
"int-tests" ) shift; cmd_int_tests $@;;
|
|
"int-tests-wb" ) shift; cmd_int_tests_white_box $@;;
|
|
"simple-tests") shift; cmd_int_tests_simple $@;;
|
|
"stop" ) cmd_stop;;
|
|
"wipe-logs" ) cmd_wipe_logs;;
|
|
"rd-sql" ) shift; cmd_rd_sql $@;;
|
|
"fake-sql" ) shift; cmd_fake_sql $@;;
|
|
"run-ci" ) shift; cmd_run_ci $@;;
|
|
"vagrant-ssh" ) shift; cmd_vagrant_ssh $@;;
|
|
"debug" ) shift; echo "Enabling debugging."; \
|
|
set -o xtrace; run_command $@;;
|
|
"clear" ) shift; cmd_clear $@;;
|
|
"run" ) shift; cmd_run $@;;
|
|
"kick-start" ) shift; cmd_kick_start $@;;
|
|
"dsvm-gate-tests" ) shift; cmd_dsvm_gate_tests $@;;
|
|
"run-fake" ) shift; cmd_run_fake $@;;
|
|
"start-fake" ) shift; cmd_start_fake $@;;
|
|
"update-projects" ) cmd_clone_projects force_update \
|
|
$REDSTACK_SCRIPTS/projects-list \
|
|
$REDSTACK_SCRIPTS/image-projects-list;;
|
|
"reset-task" ) shift; cmd_reset_task $@;;
|
|
"wipe-queues" ) shift; cmd_wipe_queues $@;;
|
|
"example-tests" ) shift; cmd_example_tests $@;;
|
|
"repl" ) shift; cmd_repl $@;;
|
|
* )
|
|
echo "'$1' not a valid command"
|
|
exit 1
|
|
esac
|
|
}
|
|
|
|
run_command $@
|