openstack/trove
Code Issues Proposed changes

Build reusable Trove guest image for dev

Browse Source 
When building the guest image for dev_mode=true, the controller IP
address is injected to the image in order for the guest-agent to
download Trove code during initialization. As a result, we have to build
guest image each time in the Trove CI because the image build relies on
the devstack host IP address.

If we could remove the dependency, we can build the image(in
devmode=true) for a specific datastore once and use that image for all
related CI jobs, which could save some time for Trove CI.

Fix the current CI issue as well.

Change-Id: If23f4f179a6ab72cfb35e4c45d55142fedb76498
This commit is contained in:
Lingxian Kong 2019-12-04 16:02:09 +13:00
parent bdd44790fb
commit 8fc0b7695d
32 changed files with 38 additions and 463 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
devstack/plugin.sh
View File

@ -281,14 +281,25 @@ function configure_trove {
iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_swift_client trove.common.clients_admin.swift_client_trove_admin
iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_glance_client trove.common.clients_admin.glance_client_trove_admin
# To avoid 'Connection timed out' error of sudo command inside the guest agent
CLOUDINIT_PATH=/etc/trove/cloudinit/${TROVE_DATASTORE_TYPE}.cloudinit
sudo mkdir -p $(dirname "$CLOUDINIT_PATH")
sudo touch "$CLOUDINIT_PATH"
sudo tee $CLOUDINIT_PATH >/dev/null <<'EOF'
# 1. To avoid 'Connection timed out' error of sudo command inside the guest agent
# 2. Config the controller IP address used by guest-agent to download Trove code during initialization (only valid for dev_mode=true).
common_cloudinit=/etc/trove/cloudinit/common.cloudinit
sudo mkdir -p $(dirname ${common_cloudinit})
sudo touch ${common_cloudinit}
sudo tee ${common_cloudinit} >/dev/null <<EOF
#cloud-config
manage_etc_hosts: "localhost"
write_files:
- path: /etc/trove/controller.conf
content: |
CONTROLLER=${SERVICE_HOST}
EOF
# NOTE(lxkong): Remove this when we support common cloud-init file for all datastores.
for datastore in "mysql" "mariadb"
do
sudo cp ${common_cloudinit} /etc/trove/cloudinit/${datastore}.cloudinit
done
}
# install_trove() - Collect source and prepare
@ -482,15 +493,6 @@ function create_guest_image {
$TROVE_MANAGE db_load_datastore_config_parameters "$TROVE_DATASTORE_TYPE" "$TROVE_DATASTORE_VERSION" \
$DEST/trove/trove/templates/$TROVE_DATASTORE_TYPE/validation-rules.json
fi
# To avoid 'Connection timed out' error of sudo command inside the guest agent.
CLOUDINIT_PATH=/etc/trove/cloudinit/${TROVE_DATASTORE_TYPE}.cloudinit
sudo mkdir -p $(dirname "$CLOUDINIT_PATH")
sudo touch "$CLOUDINIT_PATH"
sudo tee $CLOUDINIT_PATH >/dev/null <<'EOF'
#cloud-config
manage_etc_hosts: "localhost"
EOF
}
# Set up Trove management network and make configuration change.

5
integration/README.md
View File

@ -135,9 +135,8 @@ PATH_DEVSTACK_OUTPUT=/opt/stack \
- if `dev_mode=false`, the trove code for guest agent is injected into the
image at the building time. Now `dev_mode=false` is still in experimental
and not considered production ready yet.
- If `dev_mode=true`, the script assumes to be running on the host of
trove-taskmanager, otherwise, `CONTROLLER_IP` needs to be specified
explicitly.
- If `dev_mode=true`, no Trove code is injected into the guest image. The guest
agent will download Trove code during the service initialization.
For example, build a Mysql image for Ubuntu Xenial operating system:

1
integration/scripts/files/elements/ubuntu-trusty-cassandra/element-deps
View File

@ -1 +0,0 @@
ubuntu-cassandra

1
integration/scripts/files/elements/ubuntu-trusty-couchbase/element-deps
View File

@ -1 +0,0 @@
ubuntu-couchbase

1
integration/scripts/files/elements/ubuntu-trusty-couchdb/element-deps
View File

@ -1 +0,0 @@
ubuntu-couchdb

1
integration/scripts/files/elements/ubuntu-trusty-db2/element-deps
View File

@ -1 +0,0 @@
ubuntu-db2

1
integration/scripts/files/elements/ubuntu-trusty-guest/element-deps
View File

@ -1 +0,0 @@
ubuntu-guest

21
integration/scripts/files/elements/ubuntu-trusty-guest/extra-data.d/20-guest-upstart
View File

@ -1,21 +0,0 @@
#!/bin/bash
set -e
set -o xtrace
# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
# PURPOSE: stages the bootstrap file and upstart conf file while replacing variables so that guest image is properly
# configured
source $_LIB/die
[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image"
[ -n "${HOST_SCP_USERNAME}" ] || die "HOST_SCP_USERNAME needs to be set to the user for the host instance"
[ -n "${CONTROLLER_IP}" ] || die "CONTROLLER_IP needs to be set to the ip address that guests will use to contact the controller"
[ -n "${ESCAPED_PATH_TROVE}" ] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the trovestack host"
[ -n "${TROVESTACK_SCRIPTS}" ] || die "TROVESTACK_SCRIPTS needs to be set to the trove/integration/scripts dir"
[ -n "${ESCAPED_GUEST_LOGDIR}" ] || die "ESCAPED_GUEST_LOGDIR must be set to the escaped guest log dir"
sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/CONTROLLER_IP/${CONTROLLER_IP}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${TROVESTACK_SCRIPTS}/files/trove-guest.upstart.conf > ${TMP_HOOKS_PATH}/trove-guest.conf

8
integration/scripts/files/elements/ubuntu-trusty-guest/install.d/20-etc
View File

@ -1,8 +0,0 @@
#!/bin/sh
# CONTEXT: GUEST during CONSTRUCTION as ROOT
# PURPOSE: take "staged" trove-guest.conf file and put it in the init directory on guest image
dd if=/tmp/in_target.d/trove-guest.conf of=/etc/init/trove-guest.conf

117
integration/scripts/files/elements/ubuntu-trusty-guest/pre-install.d/01-trim-pkgs
View File

@ -1,117 +0,0 @@
#!/bin/bash
# CONTEXT: GUEST during CONSTRUCTION as ROOT
# PURPOSE: Install basic services and applications
set -e
set -o xtrace
export DEBIAN_FRONTEND=noninteractive
apt-get -y purge acpid\
apport\
apport-symptoms\
apt-transport-https\
apt-xapian-index\
aptitude\
at\
bash-completion\
bc\
bind9-host\
bsdmainutils\
busybox-static\
byobu\
command-not-found\
command-not-found-data\
curl\
dbus\
dmidecode\
dosfstools\
ed\
fonts-ubuntu-font-family-console\
friendly-recovery\
ftp\
fuse\
geoip-database\
groff-base\
hdparm\
info\
install-info\
iptables\
iputils-tracepath\
irqbalance\
landscape-client\
landscape-common\
language-selector-common\
laptop-detect\
libaccountsservice0\
libbind9-90\
libclass-accessor-perl\
libcwidget3\
libdns100\
libept1.4.12\
libevent-2.0-5\
libgc1c2\
libgeoip1\
libio-string-perl\
libisc95\
liblwres90\
libnfnetlink0\
libparse-debianchangelog-perl\
libparted0debian1\
libpcap0.8\
libpci3\
libpipeline1\
libpolkit-gobject-1-0\
libsasl2-modules\
libsigc++-2.0-0c2a\
libsub-name-perl\
libusb-1.0-0\
libxapian22\
lshw\
lsof\
ltrace\
man-db\
mlocate\
mtr-tiny\
nano\
ntfs-3g\
parted\
patch\
plymouth-theme-ubuntu-text\
popularity-contest\
powermgmt-base\
ppp\
pppoeconf\
python-debian\
python-gdbm\
python-pam\
python-twisted-bin\
python-xapian\
python-zope.interface\
screen\
shared-mime-info\
strace\
tasksel\
tcpdump\
telnet\
time\
tmux\
ubuntu-standard\
ufw\
update-manager-core\
update-notifier-common\
usbutils\
uuid-runtime\
w3m
# The following packages cannot be removed as they cause cloud-init to be
# uninstalled in Ubuntu 14.04
# gir1.2-glib-2.0
# libdbus-glib-1-2
# libgirepository-1.0-1
# python-chardet
# python-serial
# xz-utils
apt-get -y autoremove

1
integration/scripts/files/elements/ubuntu-trusty-mariadb/element-deps
View File

@ -1 +0,0 @@
ubuntu-mariadb

36
integration/scripts/files/elements/ubuntu-trusty-mariadb/install.d/30-mariadb
View File

@ -1,36 +0,0 @@
#!/bin/bash
# CONTEXT: GUEST during CONSTRUCTION as ROOT
# PURPOSE: Install controller base required packages
set -e
set -o xtrace
export DEBIAN_FRONTEND=noninteractive
# NOTE(vkmc): Using MariaDB repositories is required
# https://mariadb.com/kb/en/mariadb/installing-mariadb-deb-files/
apt-get --allow-unauthenticated -y install software-properties-common
get_key_robust 0xcbcb082a1bb943db
add-apt-repository 'deb http://ftp.osuosl.org/pub/mariadb/repo/10.1/ubuntu trusty main'
# Pin MariaDB repository
sudo echo -e "Package: *\nPin: origin ftp.osuosl.org\nPin-Priority: 1000" > /etc/apt/preferences.d/mariadb.pref
apt-get -y update
# The fix to make versions of percona-xtrabackup > v2.2 work with Trove
# was put into the mysql guestagent code for Mitaka. There are no current
# plans to backport so we need to make sure the guest generated when the
# tests are run for Kilo or Liberty get the 2.2 verson of PXB
if [[ $BRANCH_OVERRIDE == "stable/kilo" || $BRANCH_OVERRIDE == "stable/liberty" ]]; then
PXB_VERSION_OVERRIDE="-22"
fi
apt-get --allow-unauthenticated -y install socat percona-xtrabackup${PXB_VERSION_OVERRIDE}
apt-get --allow-unauthenticated -y install libmariadbclient18 mariadb-server
cat >/etc/mysql/conf.d/no_perf_schema.cnf <<_EOF_
[mysqld]
performance_schema = off
_EOF_

1
integration/scripts/files/elements/ubuntu-trusty-mongodb/element-deps
View File

@ -1 +0,0 @@
ubuntu-mongodb

42
integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/10-mongodb-thp
View File

@ -1,42 +0,0 @@
#!/bin/sh
set -e
set -o xtrace
export DEBIAN_FRONTEND=noninteractive
cat > /etc/init.d/disable-transparent-hugepages << '_EOF_'
#!/bin/sh
### BEGIN INIT INFO
# Provides: disable-transparent-hugepages
# Required-Start: $local_fs
# Required-Stop:
# X-Start-Before: mongod mongodb-mms-automation-agent
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: Disable Linux transparent huge pages
# Description: Disable Linux transparent huge pages, to improve
# database performance.
### END INIT INFO
case $1 in
start)
if [ -d /sys/kernel/mm/transparent_hugepage ]; then
thp_path=/sys/kernel/mm/transparent_hugepage
elif [ -d /sys/kernel/mm/redhat_transparent_hugepage ]; then
thp_path=/sys/kernel/mm/redhat_transparent_hugepage
else
return 0
fi
echo 'never' > ${thp_path}/enabled
echo 'never' > ${thp_path}/defrag
unset thp_path
;;
esac
_EOF_
chmod 755 /etc/init.d/disable-transparent-hugepages
update-rc.d disable-transparent-hugepages defaults

8
integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/20-mongodb
View File

@ -1,8 +0,0 @@
#!/bin/sh
set -e
set -o xtrace
export DEBIAN_FRONTEND=noninteractive
apt-get -y install mongodb-org=3.2.11

9
integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/25-trove-mongo-dep
View File

@ -1,9 +0,0 @@
#!/bin/bash
# CONTEXT: GUEST during CONSTRUCTION as ROOT
# PURPOSE: Install trove guest python dependencies - see trovestack functions_qemu
set -e
set -o xtrace
pip3 install pymongo>=3.0.2,!=3.1

25
integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/30-mongodb-conf
View File

@ -1,25 +0,0 @@
#!/bin/sh
set -e
set -o xtrace
# Remove the default pid file
rm -f /var/run/mongodb.pid
cat > /etc/mongod.conf << '_EOF_'
storage.dbPath: /var/lib/mongodb
security.authorization: enabled
storage.engine: wiredTiger
storage.journal.enabled: true
systemLog.destination: file
systemLog.logAppend: true
systemLog.path: /var/log/mongodb/mongod.log
_EOF_
cat > /etc/mongos.conf << '_EOF_'
systemLog.destination: file
systemLog.logAppend: true
systemLog.path: /var/log/mongodb/mongos.log
_EOF_

46
integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/41-mongod-init
View File

@ -1,46 +0,0 @@
#!/bin/sh
set -e
set -o xtrace
cat > /etc/init/mongod.conf << '_EOF_'
limit fsize unlimited unlimited
limit cpu unlimited unlimited
limit as unlimited unlimited
limit nofile 64000 64000
limit rss unlimited unlimited
limit nproc 64000 64000
kill timeout 300 # wait 300s between SIGTERM and SIGKILL.
pre-start script
mkdir -p /var/run/mongodb/
touch /var/run/mongodb/mongod.pid
chown mongodb -R /var/run/mongodb/
end script
start on runlevel [2345]
stop on runlevel [06]
script
CONF=/etc/mongod.conf
# Handle NUMA access to CPUs (SERVER-3574)
# This verifies the existence of numactl as well as testing that the command works
NUMACTL_ARGS="--interleave=all"
if which numactl >/dev/null 2>/dev/null && numactl $NUMACTL_ARGS ls / >/dev/null 2>/dev/null
then
NUMACTL="$(which numactl) -- $NUMACTL_ARGS"
DAEMON_OPTS=${DAEMON_OPTS:-"--config $CONF"}
else
NUMACTL=""
DAEMON_OPTS="-- "${DAEMON_OPTS:-"--config $CONF"}
fi
exec start-stop-daemon --start \
--chuid mongodb \
--pidfile /var/run/mongod.pid \
--make-pidfile \
--exec $NUMACTL /usr/bin/mongod $DAEMON_OPTS
end script
_EOF_

30
integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/42-mongos-init
View File

@ -1,30 +0,0 @@
#!/bin/sh
set -e
set -o xtrace
cat > /etc/init/mongos.conf << '_EOF_'
limit fsize unlimited unlimited
limit cpu unlimited unlimited
limit as unlimited unlimited
limit nofile 64000 64000
limit rss unlimited unlimited
limit nproc 64000 64000
pre-start script
mkdir -p /var/run/mongodb/
touch /var/run/mongodb/mongos.pid
chown mongodb -R /var/run/mongodb/
end script
start on runlevel [2345]
stop on runlevel [06]
script
exec start-stop-daemon --start \
--chuid mongodb \
--pidfile /var/run/mongos.pid \
--make-pidfile \
--exec /usr/bin/mongos -- --config /etc/mongos.conf
end script
_EOF_

1
integration/scripts/files/elements/ubuntu-trusty-mysql/element-deps
View File

@ -1 +0,0 @@
ubuntu-mysql

23
integration/scripts/files/elements/ubuntu-trusty-mysql/install.d/30-mysql
View File

@ -1,23 +0,0 @@
#!/bin/bash
# CONTEXT: GUEST during CONSTRUCTION as ROOT
# PURPOSE: Install controller base required packages
set -e
set -o xtrace
export DEBIAN_FRONTEND=noninteractive
# The fix to make versions of percona-xtrabackup > v2.2 work with Trove
# was put into the mysql guestagent code for Mitaka. There are no current
# plans to backport so we need to make sure the guest generated when the
# tests are run for Kilo or Liberty get the 2.2 verson of PXB
if [[ $BRANCH_OVERRIDE == "stable/kilo" || $BRANCH_OVERRIDE == "stable/liberty" ]]; then
PXB_VERSION_OVERRIDE="-22"
fi
apt-get --allow-unauthenticated -y install libmysqlclient18 mysql-server-5.6 percona-xtrabackup${PXB_VERSION_OVERRIDE}
cat >/etc/mysql/conf.d/no_perf_schema.cnf <<_EOF_
[mysqld]
performance_schema = off
_EOF_

1
integration/scripts/files/elements/ubuntu-trusty-percona/element-deps
View File

@ -1 +0,0 @@
ubuntu-percona

1
integration/scripts/files/elements/ubuntu-trusty-postgresql/element-deps
View File

@ -1 +0,0 @@
ubuntu-postgresql

1
integration/scripts/files/elements/ubuntu-trusty-pxc/element-deps
View File

@ -1 +0,0 @@
ubuntu-pxc

1
integration/scripts/files/elements/ubuntu-trusty-redis/element-deps
View File

@ -1 +0,0 @@
ubuntu-redis

1
integration/scripts/files/elements/ubuntu-trusty-vertica/element-deps
View File

@ -1 +0,0 @@
ubuntu-vertica

3
integration/scripts/files/elements/ubuntu-xenial-guest/extra-data.d/20-guest-systemd
View File

@ -13,10 +13,9 @@ source $_LIB/die
[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image"
[ -n "${HOST_SCP_USERNAME}" ] || die "HOST_SCP_USERNAME needs to be set to the user for the host instance"
[ -n "${CONTROLLER_IP}" ] || die "CONTROLLER_IP needs to be set to the ip address that guests will use to contact the controller"
[ -n "${ESCAPED_PATH_TROVE}" ] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the trovestack host"
[ -n "${TROVESTACK_SCRIPTS}" ] || die "TROVESTACK_SCRIPTS needs to be set to the trove/integration/scripts dir"
[ -n "${ESCAPED_GUEST_LOGDIR}" ] || die "ESCAPED_GUEST_LOGDIR must be set to the escaped guest log dir"
sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/CONTROLLER_IP/${CONTROLLER_IP}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${TROVESTACK_SCRIPTS}/files/trove-guest.systemd.conf > ${TMP_HOOKS_PATH}/trove-guest.service
sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${TROVESTACK_SCRIPTS}/files/trove-guest.systemd.conf > ${TMP_HOOKS_PATH}/trove-guest.service

9
integration/scripts/files/trove-guest.systemd.conf
View File

@ -8,15 +8,20 @@ Type=simple
User=GUEST_USERNAME
Group=GUEST_USERNAME
# This script is only for testing purpose for dev_mode=true, the controller
# IP address should be defined in /etc/trove/controller.conf, e.g.
# CONTROLLER=192.168.32.151
EnvironmentFile=/etc/trove/controller.conf
ExecStartPre=/bin/bash -c "sudo mkdir -p GUEST_LOGDIR ; sudo chown GUEST_USERNAME:root GUEST_LOGDIR"
# If ~/trove-installed does not exist, copy the trove source from
# the user's development environment, then touch the sentinel file
ExecStartPre=/bin/bash -c "test -e /home/GUEST_USERNAME/trove-installed || sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@CONTROLLER_IP:PATH_TROVE/ /home/GUEST_USERNAME/trove && touch /home/GUEST_USERNAME/trove-installed"
ExecStartPre=/bin/bash -c "test -e /home/GUEST_USERNAME/trove-installed || sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@$CONTROLLER:PATH_TROVE/ /home/GUEST_USERNAME/trove && touch /home/GUEST_USERNAME/trove-installed"
# If /etc/trove does not exist, create it and then copy the trove-guestagent.conf
# from /etc/trove on the user's development environment,
ExecStartPre=/bin/bash -c "test -d /etc/trove/conf.d || sudo mkdir -p /etc/trove/conf.d && sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@CONTROLLER_IP:/etc/trove/trove-guestagent.conf ~GUEST_USERNAME/ && sudo mv ~GUEST_USERNAME/trove-guestagent.conf /etc/trove/conf.d/trove-guestagent.conf"
ExecStartPre=/bin/bash -c "test -d /etc/trove/conf.d || sudo mkdir -p /etc/trove/conf.d && sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@$CONTROLLER:/etc/trove/trove-guestagent.conf ~GUEST_USERNAME/ && sudo mv ~GUEST_USERNAME/trove-guestagent.conf /etc/trove/conf.d/trove-guestagent.conf"
ExecStartPre=/bin/bash -c "sudo chown -R GUEST_USERNAME:root /etc/trove"

2
integration/scripts/functions_qemu
View File

@ -26,8 +26,6 @@ function build_vm() {
# In dev mode, the trove guest agent needs to download trove code from
# trove-taskmanager host during service initialization.
if [[ "${dev_mode}" == "true" ]]; then
host_ip=$(ip route get 8.8.8.8 | head -1 | awk '{print $7}')
export CONTROLLER_IP=${CONTROLLER_IP:-${host_ip}}
export PATH_TROVE=${PATH_TROVE}
export ESCAPED_PATH_TROVE=$(echo ${PATH_TROVE} | sed 's/\//\\\//g')
export GUEST_LOGDIR=${GUEST_LOGDIR:-"/var/log/trove/"}

58
integration/scripts/trovestack
View File

@ -1050,6 +1050,7 @@ function cmd_int_tests() {
args="$@"
fi
# Referenced in test script
export TROVE_TEST_SSH_USER=${TROVE_TEST_SSH_USER:-"ubuntu"}
export TROVE_TEST_SSH_KEY_FILE=${TROVE_TEST_SSH_KEY_FILE:-"$HOME/.ssh/id_rsa"}
@ -1285,48 +1286,17 @@ function cmd_kick_start() {
}
function cmd_gate_tests() {
ACTUAL_HOSTNAME=$(hostname -I | sed 's/[0-9a-z][0-9a-z]*:.*:[0-9a-z][0-9a-z]*//g' | sed 's/[0-9]*\.[0-9]*\.[0-9]*\.1\b//g' | sed 's/ /\n/g' | sed '/^$/d' | sort -bu | head -1)
local DATASTORE_TYPE=${1:-'mysql'}
local TEST_GROUP=${2:-${DATASTORE_TYPE}}
local HOST_SCP_USERNAME=${3:-$(whoami)}
local GUEST_USERNAME=${4:-'ubuntu'}
local CONTROLLER_IP=${5:-$ACTUAL_HOSTNAME}
# We're not using devstack-gate in Zuul v3 job
if [[ $GATE_JOB_VER == "v2" ]]; then
local ESCAPED_PATH_TROVE=${6:-'\/opt\/stack\/new\/trove'}
local ESCAPED_PATH_TROVE=${5:-'\/opt\/stack\/new\/trove'}
fi
exclaim "Running cmd_gate_tests ..."
# Sometimes in the gate the ACTUAL_HOSTNAME is blank - this code attempts to debug it
if [[ -z "${CONTROLLER_IP// }" ]]; then
echo "*** CONTROLLER_IP is blank, trying to determine actual hostname"
local hostname_part=$(hostname -I)
echo "Hostname pass 1: '$hostname_part'"
hostname_part=$(echo $hostname_part | sed 's/[0-9a-z][0-9a-z]*:.*:[0-9a-z][0-9a-z]*//g')
echo "Hostname pass 2: '$hostname_part'"
hostname_part_no_ip6=$hostname_part
hostname_part=$(echo $hostname_part | sed 's/[0-9]*\.[0-9]*\.[0-9]*\.1\b//g')
echo "Hostname pass 3: '$hostname_part'"
if [[ -z "${hostname_part// }" ]]; then
# This seems to occur when the actual hostname ends with '.1'
# If this happens, take the first one that doesn't start with '192' or '172'
hostname_part=$(echo $hostname_part_no_ip6 | sed 's/1[79]2\.[0-9]*\.[0-9]*\.1\b//g')
echo "Hostname pass 3a: '$hostname_part'"
fi
hostname_part=$(echo $hostname_part | sed 's/ /\n/g')
echo "Hostname pass 4: '$hostname_part'"
hostname_part=$(echo $hostname_part | sed '/^$/d')
echo "Hostname pass 5: '$hostname_part'"
hostname_part=$(echo $hostname_part | sort -bu)
echo "Hostname pass 6: '$hostname_part'"
hostname_part=$(echo $hostname_part | head -1)
echo "Hostname pass 7: '$hostname_part'"
CONTROLLER_IP=$hostname_part
echo "*** CONTROLLER_IP was blank (CONTROLLER_IP now set to '$CONTROLLER_IP')"
fi
export REPORT_DIRECTORY=${REPORT_DIRECTORY:=$HOME/gate-tests-report/}
export TROVE_REPORT_DIR=$HOME/gate-tests-report/
TROVESTACK_DUMP_ENV=true
@ -1337,38 +1307,14 @@ function cmd_gate_tests() {
# This var is used to ssh into the db instance during the test.
export TROVE_TEST_SSH_KEY_FILE=${SSH_DIR}/id_rsa
# To avoid 'Connection timed out' error of sudo command
CLOUDINIT_PATH=/etc/trove/cloudinit/${DATASTORE_TYPE}.cloudinit
sudo mkdir -p $(dirname "$CLOUDINIT_PATH")
sudo touch "$CLOUDINIT_PATH"
sudo tee $CLOUDINIT_PATH >/dev/null <<'EOF'
#cloud-config
manage_etc_hosts: "localhost"
EOF
# Fix iptables rules that prevent amqp connections from the devstack box to the guests
sudo iptables -D openstack-INPUT -j REJECT --reject-with icmp-host-prohibited || true
sudo chown -R $(whoami) /etc/trove
iniset $TROVE_GUESTAGENT_CONF DEFAULT rabbit_host $CONTROLLER_IP
cd $TROVESTACK_SCRIPTS
sudo -H $HTTP_PROXY pip install --upgrade pip dib-utils
local RESTART_TROVE=false
cmd_kick_start "${DATASTORE_TYPE}" "${RESTART_TROVE}"
# TODO(zhaochao): in fact, devstack also use CONTROLLER_IP to construct the endpoint urls
# so we could safely remove the following lines in the future(they're kept now for not
# introducing too many changes.
if [[ $GATE_JOB_VER == "v2" ]]; then
# Update the local swift endpoint in the catalog to use the CONTROLLER_IP instead of 127.0.0.1
SWIFT_ENDPOINTS=$(openstack $CLOUD_ADMIN_ARG endpoint list --service swift -c ID -f value)
openstack $CLOUD_ADMIN_ARG endpoint create swift public 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' --region RegionOne
openstack $CLOUD_ADMIN_ARG endpoint create swift internal 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' --region RegionOne
openstack $CLOUD_ADMIN_ARG endpoint create swift admin 'http://'$CONTROLLER_IP':8080' --region RegionOne
echo $SWIFT_ENDPOINTS | xargs -n 1 openstack $CLOUD_ADMIN_ARG endpoint delete
fi
cmd_int_tests --group=$TEST_GROUP
}

2
trove/guestagent/api.py
View File

@ -101,7 +101,7 @@ class API(object):
raise exception.GuestTimeout()
def _cast(self, method_name, version, **kwargs):
LOG.debug("Casting %s", method_name)
LOG.debug("Calling %s asynchronously", method_name)
try:
with NotificationCastWrapper(self.context, 'guest'):
cctxt = self.client.prepare(version=version)

14
trove/instance/models.py
View File

@ -24,6 +24,7 @@ import six
from novaclient import exceptions as nova_exceptions
from oslo_config.cfg import NoSuchOptError
from oslo_log import log as logging
from oslo_utils import encodeutils
from sqlalchemy import func
from trove.backup.models import Backup
@ -890,11 +891,10 @@ class BaseInstance(SimpleInstance):
instance_key = get_instance_encryption_key(self.id)
if instance_key:
files = {guest_info_file: (
"%s"
"instance_rpc_encr_key=%s\n" % (
files.get(guest_info_file),
instance_key))}
files = {
guest_info_file: ("%sinstance_rpc_encr_key=%s\n" %
(files.get(guest_info_file), instance_key))
}
if os.path.isfile(CONF.get('guest_config')):
with open(CONF.get('guest_config'), "r") as f:
@ -1811,6 +1811,10 @@ class instance_encryption_key_cache(object):
if val is None:
return val
# We need string anyway
if isinstance(val, six.binary_type):
val = encodeutils.safe_decode(val)
if len(self._lru) == self._lru_cache_size:
tail = self._lru.pop()
del self._table[tail]