Files
trove-integration/scripts/redstack
Tim Simpson 08c7ccb08f Got Reddwarf registered with Keystone so our client will connect.
* Updated redstack script to register Reddwarf endpoints with Keystone.
* Changed the test client to not combine the Nova and Reddwarf rich clients, since this no longer makes sense.
* Moved some assertions out of a test and into the utility code so they execute each time.
2012-03-14 21:32:41 -05:00

644 lines
20 KiB
Bash
Executable File

#!/usr/bin/env bash
###############################################################################
# RedStack, the Reddwarf Dev Machine Controller #
###############################################################################
# #
# This script provides all the functionality to run all the steps from #
# setting up the enviornment, resetting the nova database to running the #
# test. #
# #
###############################################################################
# Bail on errors.
set -e
if [ ! -f /integration/scripts/redstack ]; then
echo "This script is meant to run in a disposable dev environment. "
echo "Run create_vm to create such a machine."
exit 1
fi
# Load global configuration variables.
. /integration/scripts/redstack.rc
. /integration/scripts/reviews.rc
# Load functions devstack style
. /integration/scripts/functions
if [ -f $PATH_ENV_CONF ]; then
source $PATH_ENV_CONF
fi
TEST_HOME="/integration/tests"
INTEGRATION_CONF="/tmp/integration"
TESTS_CONF="$INTEGRATION_CONF/tests.conf"
###############################################################################
# Utility functions
###############################################################################
APT_GET_ARGS="-y --allow-unauthenticated --force-yes"
exclaim () {
echo "*******************************************************************************"
echo "$@"
echo "*******************************************************************************"
}
pkg_install () {
echo Installing $@...
sudo -E DEBIAN_FRONTEND=noninteractive $HTTP_PROXY apt-get $APT_GET_ARGS install $@
}
pkg_update () {
echo Updating $@...
sudo -E DEBIAN_FRONTEND=noninteractive $HTTP_PROXY apt-get $APT_GET_ARGS update $@
}
pkg_remove () {
echo Uninstalling $@...
sudo -E apt-get $APT_GET_ARGS remove $@
}
set_home_dir() {
self="${0#./}"
base="${self%/*}"
current=`pwd`
if [ "$base" = "$self" ] ; then
HOME="$current/"
elif [[ $base =~ ^/ ]]; then
HOME="$base/"
else
HOME="$current/$base/"
fi
}
set_http_proxy() {
if [ ! "${http_proxy}" = '' ]; then
HTTP_PROXY="http_proxy=$http_proxy https_proxy=$https_proxy"
fi
}
function echo_id_from_table () {
# Given a command that returns a table, prints only the id part.
echo `$@ | grep id | awk '{print $4}'`
}
function keystone_add_user() {
# Adds a user. Prints the UUID to standard out.
USER_NAME=$1
USER_PASS=$2
USER_EMAIL=$3
# Create the user "reddwarf"
USER_UUID=`keystone --endpoint http://localhost:35357/v2.0 --token be19c524ddc92109a224 user-list | grep $USER_NAME | awk '{print $2}'`
if [ -z $USER_UUID ]; then
USER_UUID=`echo_id_from_table keystone \
--endpoint http://localhost:35357/v2.0 \
--token be19c524ddc92109a224 user-create \
--name=$USER_NAME --pass="$USER_PASS" --email=$USER_EMAIL`
fi
echo $USER_UUID
}
function keystone_add_user_role() {
TENANT_UUID=$1
USER_UUID=$2
ROLE_UUID=$3
echo "keystone --endpoint http://localhost:35357/v2.0
--token be19c524ddc92109a224 user-role-add
--tenant_id $TENANT_UUID
--user $USER_UUID
--role $ROLE_UUID"
keystone --endpoint http://localhost:35357/v2.0 \
--token be19c524ddc92109a224 user-role-add \
--tenant_id $TENANT_UUID \
--user $USER_UUID \
--role $ROLE_UUID
}
keystone_manage () {
keystone --endpoint http://localhost:35357/v2.0 --token be19c524ddc92109a224 $@
}
function configure_keystone() {
exclaim "Configuring keystone..."
# Create the tenant "reddwarf".
# First we should check if these exist
REDDWARF_TENANT=`keystone --endpoint http://localhost:35357/v2.0 --token be19c524ddc92109a224 tenant-list | grep reddwarf | awk '{print $2}'`
if [ -z $REDDWARF_TENANT ]; then
REDDWARF_TENANT=`echo_id_from_table keystone --endpoint http://localhost:35357/v2.0 --token be19c524ddc92109a224 tenant-create --name=reddwarf`
fi
REDDWARF_ROLE=`keystone --endpoint http://localhost:35357/v2.0 --token be19c524ddc92109a224 role-list | grep reddwarf | awk '{print $2}'`
if [ -z "$REDDWARF_ROLE" ]; then
REDDWARF_ROLE=`echo_id_from_table keystone --endpoint http://localhost:35357/v2.0 --token be19c524ddc92109a224 role-create --name=reddwarf`
fi
REDDWARF_USER=`keystone_add_user reddwarf REDDWARF-PASS reddwarf@example.com`
keystone_add_user_role $REDDWARF_TENANT $REDDWARF_USER $REDDWARF_ROLE
# TODO: Restrict permissions.
REDDWARF_USER=`keystone_add_user radmin radmin radmin@example.com`
keystone_add_user_role $REDDWARF_TENANT $REDDWARF_USER $REDDWARF_ROLE
REDDWARF_USER=`keystone_add_user Boss admin boss@example.com`
keystone_add_user_role $REDDWARF_TENANT $REDDWARF_USER $REDDWARF_ROLE
REDDWARF_USER=`keystone_add_user chunk chunk chunk@example.com`
keystone_add_user_role $REDDWARF_TENANT $REDDWARF_USER $REDDWARF_ROLE
REDDWARF_USER=`keystone_add_user daffy daffy daffy@example.com`
keystone_add_user_role $REDDWARF_TENANT $REDDWARF_USER $REDDWARF_ROLE
REDDWARF_USER=`keystone_add_user examples examples examples@example.com`
keystone_add_user_role $REDDWARF_TENANT $REDDWARF_USER $REDDWARF_ROLE
echo "
REDDWARF_TENANT=$REDDWARF_TENANT
REDDWARF_USER=$REDDWARF_USER
REDDWARF_ROLE=$REDDWARF_ROLE
" > $PATH_ENV_CONF
echo "
# REDDWARF_TENANT=$REDDWARF_TENANT
# REDDWARF_USER=$REDDWARF_USER
# REDDWARF_ROLE=$REDDWARF_ROLE"
echo "Checking login..."
# Now attempt a login
curl -d '{"auth":{"passwordCredentials":{"username": "reddwarf", "password": "REDDWARF-PASS"},"tenantName":"reddwarf"}}' \
-H "Content-type: application/json" http://localhost:35357/v2.0/tokens
# Register reddwarf service.
SERVICE_UUID=`keystone_manage service-list | grep reddwarf | awk '{print $2}'`
if [ -z $SERVICE_UUID ]; then
OS_USERNAME=reddwarf OS_PASSWORD=REDDWARF-PASS \
keystone_manage service-create --name=reddwarf \
--type=dbaas \
--description="Reddwarf"
fi
}
###############################################################################
###############################################################################
# Install all the required dependencies
###############################################################################
install_packages_1() {
# Called before devstack
exclaim 'Updating dependencies (part 1a)...'
pkg_update
exclaim 'Installing dependencies (part 1b)...'
pkg_install git-core kvm-pxe ubuntu-vm-builder
}
add_to_keystone_conf() {
echo "
catalog.RegionOne.reddwarf.publicURL = http://10.0.2.15:8779/v0.1
catalog.RegionOne.reddwarf.adminURL = http://10.0.2.15:8779/v0.1
catalog.RegionOne.reddwarf.internalURL = http://10.0.2.15:8779/v0.1
catalog.RegionOne.reddwarf.name = 'Reddwarf'
" >> $PATH_KEYSTONE/etc/default_catalog.templates
}
install_packages_2() {
# Called after devstack.
exclaim "Installing dependencies (part 2)..."
# The default version of pip Ubuntu installs is old.
cd /tmp
sudo pip install --upgrade pip
pkg_install python-pexpect
sudo pip install proboscis
cmd_stop_deps
add_to_keystone_conf
cmd_start_deps
}
install_devstack() {
exclaim "Installing devstack..."
# Installs devstack (if needed).
if [ ! -d $PATH_DEVSTACK_SRC ]; then
echo "DevStack not in a shared folder, cloning from git."
mkdir -p $PATH_DEVSTACK_SRC
git clone git://github.com/openstack-dev/devstack.git $PATH_DEVSTACK_SRC
fi
}
install_devstack_code() {
exclaim "Installing devstack projects..."
# Ensures present user can get to the devstack dirs
sudo mkdir -p $PATH_DEVSTACK_OUTPUT
if [ ! -w $PATH_DEVSTACK_OUTPUT ]; then
sudo chown `whoami` $PATH_DEVSTACK_OUTPUT
fi
# Clones all of the code to where devstack expects it to be
cd $PATH_DEVSTACK_OUTPUT
for project in glance horizon keystone nova python-keystoneclient python-novaclient python-quantumclient
do
if [ ! -d $project ]; then
echo "Creating a new clone of $project..."
git clone git://github.com/openstack/$project
else
echo "$project was already cloned or exists in a shared folder, ignoring..."
fi
done
}
install_reviews_on_top_of_devstack() {
exclaim "Putting gerrit review code on top of the existing devstack code"
run_review_for nova $PATH_NOVA $REVIEW_NOVA
run_review_for python-novaclient $PATH_PYTHON_NOVACLIENT $REVIEW_PYTHON_NOVACLIENT
run_review_for keystone $PATH_KEYSTONE $REVIEW_KEYSTONE
run_review_for python-keystoneclient $PATH_KEYSTONECLIENT $REVIEW_PYTHON_KEYSTONECLIENT
run_review_for glance $PATH_GLANCE $REVIEW_GLANCE
}
run_review_for() {
# Splits based on colon in the REVIEW_ARG and pulls from
GIT_NAME=$1
PATH_ARG=$2
REVIEW_ARG=$3
for review in `echo $REVIEW_ARG| tr ":" "\n"`
do
# This should be the ref spec for what we pull
pushd $PATH_ARG
git pull https://review.openstack.org/p/openstack/$GIT_NAME refs/changes/$review
popd
done
}
run_devstack() {
exclaim "Running devstack..."
cd $PATH_DEVSTACK_SRC
echo "
MYSQL_PASSWORD=$MYSQL_PASSWORD
RABBIT_PASSWORD=$RABBIT_PASSWORD
SERVICE_TOKEN=$SERVICE_TOKEN
ADMIN_PASSWORD=$ADMIN_PASSWORD
SERVICE_PASSWORD=$SERVICE_PASSWORD" > localrc
./stack.sh
}
ensure_local_sources() {
# this method will only clone if the sources dont exist,
# or if RECLONE=yes is set
git_clone $REPO_REDDWARF $PATH_REDDWARF $BRANCH_REDDWARF
git_clone $REPO_REDDWARFCLIENT $PATH_PYTHON_REDDWARFCLIENT $BRANCH_REDDWARFCLIENT
}
cmd_install() {
install_packages_1
install_devstack
install_devstack_code
install_reviews_on_top_of_devstack
run_devstack
ensure_local_sources
install_packages_2
}
###############################################################################
# Build
###############################################################################
setup_py() {
# Calls setup.py in the given directory.
echo "CD into $1"
pushd $1
sudo python setup.py develop
popd
}
cmd_build() {
# Run setup.py for all projects.
exclaim "Calling setup for all dependencies..."
setup_py $PATH_NOVA
setup_py $PATH_PYTHON_NOVACLIENT
setup_py $PATH_PYTHON_REDDWARFCLIENT
setup_py $PATH_REDDWARF
}
###############################################################################
# Build the image
###############################################################################
generate_empty_passphrase_ssh_key() {
exclaim "generating a empty passphrase DEV ONLY rsa key"
HOMEDIR=$1
expect -c "
spawn /usr/bin/ssh-keygen -f $HOMEDIR/.ssh/id_rsa -q
expect \"empty for no passphrase\"
send \n
expect assphrase
send \n
expect eof"
}
function get_glance_id () {
echo `$@ | awk '{print $6}'`
}
function upload_image_to_glance() {
exclaim "uploading the image to glance."
IMAGE_NAME=$1
REDDWARF_TOKEN=$2
IMAGE_LOC=$3
UBUNTU_DISTRO=$4
GLANCE_IMAGEID=`get_glance_id glance add --silent-upload name="$IMAGE_NAME" is_public=true \
container_format=ovf disk_format=qcow2 \
distro="$UBUNTU_DISTRO" -A $REDDWARF_TOKEN < $IMAGE_LOC`
#TODO(hub-cap): Upload this sucker to our database.
# This should be a reddwarf-manage function
echo "update service_images set image_id = '$GLANCE_IMAGEID';"
}
function build_vm() {
exclaim "Actually building the image, this can take up to 15 minutes"
HOMEDIR=$1
USERNAME=$2
UBUNTU_DISTRO_NAME=$3
VM_PATH=$4
# Create the guest with the specific files
mkdir -p $HOMEDIR/.ssh
if [ ! -e $HOMEDIR/.ssh/id_rsa.pub ]
then
pkg_install expect
# Generate a public key
generate_empty_passphrase_ssh_key $HOMEDIR
fi
# Generate the bootstrap_init file for --firstboot use in the ubuntu-vm-builder
# This file copies the current reddwarf codebase into each spawned image
# and executes the guestagent binfile, starting the service.
# Please note, this is ONLY for dev and should not be considered production ready.
echo "#!/usr/bin/env bash
# Copies all the reddwarf code to the guest image
sudo -u $USERNAME scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -r 10.0.0.1:${PATH_REDDWARF}/ ~${USERNAME}
# Install all the dependencies
pip install -r /home/${USERNAME}/reddwarf_lite/tools/pip-requires
# Starts the reddwarf guestagent
/home/${USERNAME}/reddwarf_lite/bin/reddwarf-guestagent --configfile=/home/${USERNAME}/reddwarf_lite/etc/reddwarf/reddwarf-guestagent.conf.sample
" > /tmp/bootstrap_init.sh
# Now put the pub key in this machines auth keys
# so the vm can log in to the host (scp from bootstrap_init.sh)
# This will also enable us to log in to the dev instances w/o a password
is_in_keyfile=`cat $HOMEDIR/.ssh/id_rsa.pub |grep -f - $HOMEDIR/.ssh/authorized_keys |wc -l`
if [ $is_in_keyfile == 0 ]
then
echo "Adding keyfile to authorized_keys, it does not yet exist"
cat $HOMEDIR/.ssh/id_rsa.pub >> $HOMEDIR/.ssh/authorized_keys
fi
COPY_FILE=/tmp/guest-agent-files.txt
rm -fr $COPY_FILE
# These will be the way the firstboot script phones home to get the latest guest via scp.
# See bootstrap_init.sh for more info on what it does
# man ubuntu-vm-builder for more infomration on the --copy flag
echo "$HOMEDIR/.ssh/id_rsa.pub $HOMEDIR/.ssh/id_rsa.pub" >> $COPY_FILE
echo "$HOMEDIR/.ssh/id_rsa $HOMEDIR/.ssh/id_rsa" >> $COPY_FILE
echo "$HOMEDIR/.ssh/authorized_keys $HOMEDIR/.ssh/authorized_keys" >> $COPY_FILE
#build a qemu image
sudo ubuntu-vm-builder qemu $UBUNTU_DISTRO_NAME --addpkg vim \
--addpkg mysql-server --addpkg openssh-server \
--addpkg gcc --addpkg python-dev \
--addpkg libxml2-dev --addpkg libxslt-dev \
--addpkg python-setuptools --addpkg python-pip \
--copy $COPY_FILE --user $USERNAME --pass $USERNAME \
--firstboot /tmp/bootstrap_init.sh -d $VM_PATH
}
cmd_build_image() {
exclaim "Building a image for use with development and integration tests."
USERNAME=`whoami`
HOMEDIR=`getent passwd $USERNAME | cut -d: -f6`
VM_PATH=/tmp/oneiric_mysql_image
UBUNTU_DISTRO=ubuntu_oneiric
UBUNTU_DISTRO_NAME=oneiric
# If the path doesnt exist, build it, otherwise just upload it
if [ ! -d $VM_PATH ]
then
build_vm $HOMEDIR $USERNAME $UBUNTU_DISTRO_NAME $VM_PATH
fi
QCOW_IMAGE=`find $VM_PATH -name '*.qcow2'`
REDDWARF_TOKEN=`retrieve_token reddwarf REDDWARF-PASS reddwarf`
# Now upload it
upload_image_to_glance "${UBUNTU_DISTRO_NAME}_mysql_image" $REDDWARF_TOKEN $QCOW_IMAGE $UBUNTU_DISTRO
}
###############################################################################
# Run Unit Tests
###############################################################################
cmd_unit_tests() {
exclaim "Running Reddwarf Unit Tests..."
$PATH_REDDWARF/run_tests.sh -N
}
###############################################################################
# Start various OpenStack daemons interactively in a screen session
###############################################################################
cmd_start_deps() {
cd $PATH_DEVSTACK_SRC
./rejoin-stack.sh
}
cmd_stop_deps() {
cd $PATH_DEVSTACK_SRC
screen -S stack -X quit
rm *.pid*
}
###############################################################################
# Initialize Reddwarf
###############################################################################
fix_rd_configfile() {
cd $PATH_REDDWARF
cp etc/reddwarf/reddwarf.conf.sample /tmp
# Do some munging to get it a-working
# Tenant is no longer needed from the config, but we are leaving it in
# until we have a better example of stuff to sed out.
# sed -i.bak "s/reddwarf_tenant_id=.*/reddwarf_tenant_id=$REDDWARF_TENANT/g" /tmp/reddwarf.conf.sample
}
rd_manage() {
cd $PATH_REDDWARF
bin/reddwarf-manage --config-file=/tmp/reddwarf.conf.sample $@
}
cmd_initialize() {
exclaim 'Initializing...'
#run_devstack
mkdir -p $PATH_INTEGRATION_CONF
exclaim "Creating Keystone users..."
configure_keystone
exclaim "Making a temporary reddwarf config file..."
fix_rd_configfile
exclaim "Initializing the Reddwarf Database..."
rd_manage db_sync
}
###############################################################################
# Start Reddwarf specific daemons interactively in a screen session
###############################################################################
function screen_it {
echo "Starting $@..."
screen -S reddwarf -X screen -t $1
screen -S reddwarf -p $1 -X stuff "$2$NL"
}
cmd_start() {
if [ ! -f ~/.screenrc ]; then
cat >~/.screenrc <<EOF
hardstatus on
hardstatus alwayslastline
hardstatus string "%{.bW}%-w%{.rW}%n %t%{-}%+w %=%{..G}%H %{..Y}%d/%m %c"
defscrollback 1024
vbell off
startup_message off
EOF
fi
NL=`echo -ne '\015'`
screen -d -m -S reddwarf -t reddwarf
sleep 1
screen_it reddwarf "cd $PATH_REDDWARF; bin/reddwarf-server --config-file=/tmp/reddwarf.conf.sample"
screen -S reddwarf -x
}
###############################################################################
# Stop any active Reddwarf screen session
###############################################################################
###############################################################################
cmd_stop() {
screen -S reddwarf -X quit
rm *.pid*
}
###############################################################################
# Run Integration Tests
###############################################################################
cmd_int_tests() {
exclaim "Running Reddwarf Integration Tests..."
export TEST_CONF=/integration/scripts/conf/test.conf
NEW_PATH=$PYTHONPATH:$PATH_REDDWARF:$PATH_REDDWARF_INT_TESTS/integration
echo "Python path : $NEW_PATH"
export PYTHONPATH=$NEW_PATH
# By default use the blackbox group.
if [ $# -lt 1 ]; then
args="--group=blackbox"
else
args="$@"
fi
# -- verbose makes it prettier.
# -- logging-clear-handlers keeps the novaclient and other things from
# spewing logs to stdout.
args="$INT_TEST_OPTIONS -B $PATH_REDDWARF_INT_TESTS/integration/int_tests.py --verbose --logging-clear-handlers $args"
echo "python $args"
python $args
}
###############################################################################
# Process the user provided command and run the appropriate command
###############################################################################
# Lets not run this as the root user
if [ $EUID -eq 0 ]; then
echo "You are running this script as root. You need to run as a regular user"
exit 1
fi
# Set this to exit immediately on error
set -o errexit
set_home_dir
set_http_proxy
print_usage() {
echo "Usage: $0 [command]"
echo "
Commands :
install - Install all the required dependencies
build - Build the packages (including the agent)
initialize - Cleans nova database, sets everything up.
build-image - Builds the vm image.
unit-tests - Run the unit tests.
start-deps - Start or resume daemons Reddwarf depends on.
stop-deps - Kill daemons Reddwarf depends on.
start - Start or resume daemons Reddwarf depends on.
int-tests - Runs the integration tests (requires all daemons).
stop - Kill daemons Reddwarf depends on.
"
exit 1
}
# Print the available commands
if [ $# -lt 1 ]; then
print_usage
fi
case "$1" in
"install" ) cmd_install;;
"build" ) cmd_build;;
"initialize" ) cmd_initialize;;
"build-image" ) cmd_build_image;;
"unit-tests" ) cmd_unit_tests;;
"start-deps" ) cmd_start_deps;;
"stop-deps" ) cmd_stop_deps;;
"start" ) cmd_start;;
"int-tests" ) shift; cmd_int_tests $@;;
"stop" ) cmd_stop;;
* )
echo "'$1' not a valid command"
exit 1
esac