openstack/trove
Code Issues Proposed changes

Initial work to support Xenial

Browse Source 
The main goal of this change is to add support for Xenial to
trove/integration and ensure the voting check jobs continue
to pass when switched to Xenial.

- It was decided that Trusty should continue to be supported
to aid in the developer transition to Xenial. Therefore,
the elements have been organized to accomplish this by making
"trusty" and "xenial" specific elements extend the already
existing "ubuntu" elements.
- We tried to do the initial work without impacting the core
Trove code therefore some choices were made - for example,
cluster delete doesn't work for PXC on Xenial (see [1]) so
the test that does this is skipped for PXC.
- Only Mysql, PXC, Postgresql and Redis were fixed for
Xenial and tested as these are the only voting jobs
currently. Future work should sort out the elements for the
rest of the datastores. Note that all of the datastores should
continue to function on Trusty.
- Previously, the "DISTRO" variable was hardcoded to "ubuntu".
This is no longer the case; DISTRO and RELEASE variables will
be automatically set to whatever the base OS is (for example,
if you are running trovestack on Xenial, DISTRO will be set
to "ubuntu" and RELEASE will be set to "xenial". These can
be overriden if needed.

[1] https://bugs.launchpad.net/trove/+bug/1645096

Change-Id: I8e1de6ef31f969ccee88c334a0d5ed03aabd1b51
Depends-On: I3831de12c999ef8818e80ecdb29f1d86ff8cd5c8
This commit is contained in:
Doug Shelley 2016-11-03 10:21:25 +00:00
parent 731ca1bdfd
commit 8bd7aa81a1
37 changed files with 259 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
integration/scripts/files/elements/ubuntu-mysql/pre-install.d/10-percona-apt-key
View File

@ -7,7 +7,7 @@ set -e
set -o xtrace
[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image"
[ -n "${RELEASE}" ] || die "RELEASE must be set to either Precise or Quantal"
[ -n "${RELEASE}" ] || die "RELEASE must be set to a valid Ubuntu release (e.g. trusty)"
# Add Percona GPG key
mkdir -p /home/${GUEST_USERNAME}/.gnupg

4
integration/scripts/files/elements/ubuntu-postgresql/pre-install.d/10-postgresql-repo
View File

@ -3,8 +3,10 @@
set -e
set -o xtrace
[ -n "${RELEASE}" ] || die "RELEASE must be set to a valid Ubuntu release (e.g. trusty)"
cat <<EOL > /etc/apt/sources.list.d/postgresql.list
deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main
deb http://apt.postgresql.org/pub/repos/apt/ ${RELEASE}-pgdg main
EOL
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -

1
integration/scripts/files/elements/ubuntu-trusty-cassandra/element-deps Normal file
View File

@ -0,0 +1 @@
ubuntu-cassandra

1
integration/scripts/files/elements/ubuntu-trusty-couchbase/element-deps Normal file
View File

@ -0,0 +1 @@
ubuntu-couchbase

1
integration/scripts/files/elements/ubuntu-trusty-couchdb/element-deps Normal file
View File

@ -0,0 +1 @@
ubuntu-couchdb

1
integration/scripts/files/elements/ubuntu-trusty-db2/element-deps Normal file
View File

@ -0,0 +1 @@
ubuntu-db2

1
integration/scripts/files/elements/ubuntu-trusty-guest/element-deps Normal file
View File

@ -0,0 +1 @@
ubuntu-guest

0
integration/scripts/files/elements/ubuntu-guest/extra-data.d/20-guest-upstart → integration/scripts/files/elements/ubuntu-trusty-guest/extra-data.d/20-guest-upstart
View File

0
integration/scripts/files/elements/ubuntu-guest/install.d/20-etc → integration/scripts/files/elements/ubuntu-trusty-guest/install.d/20-etc
View File

0
integration/scripts/files/elements/ubuntu-guest/pre-install.d/01-trim-pkgs → integration/scripts/files/elements/ubuntu-trusty-guest/pre-install.d/01-trim-pkgs
View File

1
integration/scripts/files/elements/ubuntu-trusty-mariadb/element-deps Normal file
View File

@ -0,0 +1 @@
ubuntu-mariadb

1
integration/scripts/files/elements/ubuntu-trusty-mongodb/element-deps Normal file
View File

@ -0,0 +1 @@
ubuntu-mongodb

1
integration/scripts/files/elements/ubuntu-trusty-mysql/element-deps Normal file
View File

@ -0,0 +1 @@
ubuntu-mysql

0
integration/scripts/files/elements/ubuntu-mysql/install.d/30-mysql → integration/scripts/files/elements/ubuntu-trusty-mysql/install.d/30-mysql
View File

1
integration/scripts/files/elements/ubuntu-trusty-percona/element-deps Normal file
View File

@ -0,0 +1 @@
ubuntu-percona

1
integration/scripts/files/elements/ubuntu-trusty-postgresql/element-deps Normal file
View File

@ -0,0 +1 @@
ubuntu-postgresql

1
integration/scripts/files/elements/ubuntu-trusty-pxc/element-deps Normal file
View File

@ -0,0 +1 @@
ubuntu-pxc

1
integration/scripts/files/elements/ubuntu-trusty-redis/element-deps Normal file
View File

@ -0,0 +1 @@
ubuntu-redis

1
integration/scripts/files/elements/ubuntu-trusty-vertica/element-deps Normal file
View File

@ -0,0 +1 @@
ubuntu-vertica

1
integration/scripts/files/elements/ubuntu-xenial-guest/element-deps Normal file
View File

@ -0,0 +1 @@
ubuntu-guest

22
integration/scripts/files/elements/ubuntu-xenial-guest/extra-data.d/20-guest-systemd Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
set -e
set -o xtrace
# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
# PURPOSE: stages the bootstrap file and upstart conf file while replacing variables so that guest image is properly
# configured
source $_LIB/die
[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image"
[ -n "${HOST_SCP_USERNAME}" ] || die "HOST_SCP_USERNAME needs to be set to the user for the host instance"
[ -n "${CONTROLLER_IP}" ] || die "CONTROLLER_IP needs to be set to the ip address that guests will use to contact the controller"
[ -n "${ESCAPED_PATH_TROVE}" ] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the trovestack host"
[ -n "${TROVESTACK_SCRIPTS}" ] || die "TROVESTACK_SCRIPTS needs to be set to the trove/integration/scripts dir"
[ -n "${ESCAPED_GUEST_LOGDIR}" ] || die "ESCAPED_GUEST_LOGDIR must be set to the escaped guest log dir"
sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/CONTROLLER_IP/${CONTROLLER_IP}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${TROVESTACK_SCRIPTS}/files/trove-guest.systemd.conf > ${TMP_HOOKS_PATH}/trove-guest.service

10
integration/scripts/files/elements/ubuntu-xenial-guest/install.d/20-etc Executable file
View File

@ -0,0 +1,10 @@
#!/bin/sh
# CONTEXT: GUEST during CONSTRUCTION as ROOT
# PURPOSE: take "staged" trove-guest.conf file and put it in the init directory on guest image
dd if=/tmp/in_target.d/trove-guest.service of=/etc/systemd/system/trove-guest.service
systemctl enable trove-guest.service

90
integration/scripts/files/elements/ubuntu-xenial-guest/pre-install.d/01-trim-pkgs Executable file
View File

@ -0,0 +1,90 @@
#!/bin/bash
# CONTEXT: GUEST during CONSTRUCTION as ROOT
# PURPOSE: Install basic services and applications
set -e
set -o xtrace
export DEBIAN_FRONTEND=noninteractive
apt-get -y purge acpid\
apport\
apport-symptoms\
apt-transport-https\
aptitude\
at\
bash-completion\
bc\
bind9-host\
bsdmainutils\
busybox-static\
byobu\
command-not-found\
command-not-found-data\
curl\
dbus\
dmidecode\
dosfstools\
ed\
fonts-ubuntu-font-family-console\
friendly-recovery\
ftp\
fuse\
geoip-database\
groff-base\
hdparm\
info\
install-info\
iptables\
iputils-tracepath\
irqbalance\
language-selector-common\
libaccountsservice0\
libevent-2.0-5\
libgeoip1\
libnfnetlink0\
libpcap0.8\
libpci3\
libpipeline1\
libpolkit-gobject-1-0\
libsasl2-modules\
libusb-1.0-0\
lshw\
lsof\
ltrace\
man-db\
mlocate\
mtr-tiny\
nano\
ntfs-3g\
parted\
patch\
plymouth-theme-ubuntu-text\
popularity-contest\
powermgmt-base\
ppp\
screen\
shared-mime-info\
strace\
tcpdump\
telnet\
time\
tmux\
ubuntu-standard\
ufw\
update-manager-core\
update-notifier-common\
usbutils\
uuid-runtime\
# The following packages cannot be removed as they cause cloud-init to be
# uninstalled in Ubuntu 14.04
# gir1.2-glib-2.0
# libdbus-glib-1-2
# libgirepository-1.0-1
# python-chardet
# python-serial
# xz-utils
apt-get -y autoremove

1
integration/scripts/files/elements/ubuntu-xenial-mysql/element-deps Normal file
View File

@ -0,0 +1 @@
ubuntu-mysql

32
integration/scripts/files/elements/ubuntu-xenial-mysql/install.d/30-mysql Executable file
View File

@ -0,0 +1,32 @@
#!/bin/bash
# CONTEXT: GUEST during CONSTRUCTION as ROOT
# PURPOSE: Install controller base required packages
set -e
set -o xtrace
export DEBIAN_FRONTEND=noninteractive
add-apt-repository 'deb http://archive.ubuntu.com/ubuntu trusty universe'
apt-get -y update
apt-get -y install mysql-client-5.6 mysql-server-5.6 percona-xtrabackup${PXB_VERSION_OVERRIDE}
cat >/etc/mysql/conf.d/no_perf_schema.cnf <<_EOF_
[mysqld]
performance_schema = off
_EOF_
mv /etc/mysql/my.cnf.fallback /etc/mysql/my.cnf
chown mysql:mysql /etc/mysql/my.cnf
cat >/etc/mysql/my.cnf <<_EOF_
[mysql]
!includedir /etc/mysql/conf.d/
_EOF_
rm /etc/init/mysql.conf
systemctl daemon-reload
systemctl enable mysql

1
integration/scripts/files/elements/ubuntu-xenial-postgresql/element-deps Normal file
View File

@ -0,0 +1 @@
ubuntu-postgresql

7
integration/scripts/files/elements/ubuntu-xenial-postgresql/install.d/11-fix-init-script Executable file
View File

@ -0,0 +1,7 @@
#!/bin/sh
set -e
set -o xtrace
sed -i "s/PIDFile=\/var\/run\/postgresql\/%i.pid/PIDFile=\/var\/run\/postgresql\/postgresql.pid/" /lib/systemd/system/postgresql@.service

1
integration/scripts/files/elements/ubuntu-xenial-pxc/element-deps Normal file
View File

@ -0,0 +1 @@
ubuntu-pxc

15
integration/scripts/files/elements/ubuntu-xenial-pxc/install.d/31-fix-my-cnf Executable file
View File

@ -0,0 +1,15 @@
#!/bin/sh
# CONTEXT: GUEST during CONSTRUCTION as ROOT
# PURPOSE: Install controller base required packages
set -e
set -o xtrace
mv /etc/mysql/my.cnf.fallback /etc/mysql/my.cnf
chown mysql:mysql /etc/mysql/my.cnf
cat >/etc/mysql/my.cnf <<_EOF_
[mysql]
>
!includedir /etc/mysql/conf.d/
_EOF_

1
integration/scripts/files/elements/ubuntu-xenial-redis/element-deps Normal file
View File

@ -0,0 +1 @@
ubuntu-redis

28
integration/scripts/files/elements/ubuntu-xenial-redis/install.d/11-fix-init-file Executable file
View File

@ -0,0 +1,28 @@
#!/bin/sh
set -e
set -o xtrace
cat > /lib/systemd/system/redis-server.service << '_EOF_'
[Unit]
Description=Advanced key-value store
After=network.target
[Service]
Type=forking
PIDFile=/var/run/redis/redis-server.pid
User=redis
Group=redis
Environment=statedir=/var/run/redis
PermissionsStartOnly=true
ExecStartPre=/bin/mkdir -p ${statedir}
ExecStartPre=/bin/chown -R redis:redis ${statedir}
ExecStart=/usr/bin/redis-server /etc/redis/redis.conf
ExecReload=/bin/kill -USR2 $MAINPID
ExecStop=/usr/bin/redis-cli shutdown
Restart=always
[Install]
WantedBy=multi-user.target
_EOF_

28
integration/scripts/functions
View File

@ -193,30 +193,36 @@ GetOSVersion() {
function GetDistro() {
GetOSVersion
if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) ]]; then
# 'Everyone' refers to Ubuntu / Debian releases by the code name adjective
DISTRO=$os_CODENAME
DISTRO_NAME=ubuntu
DISTRO_RELEASE=$os_CODENAME
elif [[ "$os_VENDOR" =~ (Fedora) ]]; then
# For Fedora, just use 'f' and the release
DISTRO="f$os_RELEASE"
DISTRO_NAME=fedora
DISTRO_RELEASE=$os_RELEASE
elif [[ "$os_VENDOR" =~ (openSUSE) ]]; then
DISTRO="opensuse-$os_RELEASE"
DISTRO_NAME=opensuse
DISTRO_RELEASE=$os_RELEASE
elif [[ "$os_VENDOR" =~ (SUSE LINUX) ]]; then
DISTRO_NAME=sle
# For SLE, also use the service pack
if [[ -z "$os_UPDATE" ]]; then
DISTRO="sle${os_RELEASE}"
DISTRO_RELEASE=$os_RELEASE
else
DISTRO="sle${os_RELEASE}sp${os_UPDATE}"
DISTRO_RELEASE="${os_RELEASE}sp${os_UPDATE}"
fi
elif [[ "$os_VENDOR" =~ (Red Hat) || "$os_VENDOR" =~ (CentOS) ]]; then
# Drop the . release as we assume it's compatible
DISTRO="rhel${os_RELEASE::1}"
DISTRO_NAME=rhel
DISTRO_RELEASE=${os_RELEASE::1}
elif [[ "$os_VENDOR" =~ (XenServer) ]]; then
DISTRO="xs$os_RELEASE"
DISTRO_NAME=xs
DISTRO_RELEASE=$os_RELEASE
else
# Catch-all for now is Vendor + Release + Update
DISTRO="$os_VENDOR-$os_RELEASE.$os_UPDATE"
DISTRO_NAME=$os_VENDOR
DISTRO_RELEASE=$os_RELEASE.$os_UPDATE
fi
export DISTRO
export DISTRO_NAME
export DISTRO_RELEASE
}

8
integration/scripts/functions_qemu
View File

@ -19,14 +19,8 @@ function build_vm() {
manage_ssh_keys
if [ $DISTRO == 'ubuntu' ]; then
export RELEASE=trusty
export DIB_RELEASE=$RELEASE
export DIB_CLOUD_IMAGES=cloud-images.ubuntu.com
# Use the apt sources.list on the build host, its almost always preferred
if [ -f /etc/apt/sources.list ]; then
export DIB_APT_SOURCES=/etc/apt/sources.list
EXTRA_ELEMENTS="apt-sources apt-conf-dir"
fi
fi
if [ $DISTRO == 'fedora' ]; then
EXTRA_ELEMENTS=selinux-permissive
@ -54,7 +48,7 @@ function build_vm() {
local QEMU_IMG_OPTIONS=$(! $(qemu-img | grep -q 'version 1') && echo "--qemu-img-options compat=0.10")
${PATH_DISKIMAGEBUILDER}/bin/disk-image-create -a amd64 -o "${VM}" \
-x ${QEMU_IMG_OPTIONS} ${DISTRO} ${EXTRA_ELEMENTS} vm heat-cfntools \
cloud-init-datasources ${DISTRO}-guest ${DISTRO}-${SERVICE_TYPE}
cloud-init-datasources ${DISTRO}-${RELEASE}-guest ${DISTRO}-${RELEASE}-${SERVICE_TYPE}
}
function build_guest_image() {

10
integration/scripts/trovestack
View File

@ -48,6 +48,12 @@ function get_default_host_ip() {
. $TROVESTACK_SCRIPTS/functions
. $TROVESTACK_SCRIPTS/functions_qemu
# Pre-set DISTRO and RELEASE variables based on host OS
# Can be overridden by env vars DISTRO and RELEASE
GetDistro
export DISTRO=${DISTRO:-$DISTRO_NAME}
export RELEASE=${RELEASE:-$DISTRO_RELEASE}
# Load global configuration variables.
. $TROVESTACK_SCRIPTS/trovestack.rc
. $TROVESTACK_SCRIPTS/reviews.rc
@ -266,7 +272,8 @@ function install_prep_packages() {
if is_fedora; then
pkg_install git gettext
else
pkg_install git-core kvm-ipxe gettext
#pkg_install git-core kvm-ipxe gettext
pkg_install git-core gettext
fi
sudo -H $HTTP_PROXY pip install --upgrade pip dib-utils
}
@ -1329,7 +1336,6 @@ function cmd_repl() {
INT_TEST_OPTIONS=-i cmd_int_tests_white_box --repl --group=_does_not_exist_ $@
}
###############################################################################
# Process the user provided command and run the appropriate command
###############################################################################

3
integration/scripts/trovestack.rc
View File

@ -103,9 +103,6 @@ SWIFT_HASH=12go358snjw24501
SWIFT_DATA_DIR=${DATA_DIR}/swift
SWIFT_DISK_IMAGE=${SWIFT_DATA_DIR}/drives/images/swift.img
DISTRO=${DISTRO:-ubuntu}
#DISTRO=fedora
# The following values can be used to tweak how devstack sets
# up Trove. If not explicitly set, the defaults in the code are used.
# To make changes without modifying the repo, add these variables

1
trove/tests/scenario/runners/__init__.py
View File

@ -1,2 +1,3 @@
BUG_EJECT_VALID_MASTER = 1622014
BUG_WRONG_API_VALIDATION = 1498573
BUG_STOP_DB_IN_CLUSTER = 1645096

6
trove/tests/scenario/runners/cluster_actions_runners.py
View File

@ -22,6 +22,8 @@ import time as timer
from trove.common import exception
from trove.common.utils import poll_until
from trove.tests.scenario.helpers.test_helper import DataType
from trove.tests.scenario import runners
from trove.tests.scenario.runners.test_runners import SkipKnownBug
from trove.tests.scenario.runners.test_runners import TestRunner
from trove.tests.util.check import TypeCheck
from troveclient.compat import exceptions
@ -423,6 +425,10 @@ class PxcClusterActionsRunner(ClusterActionsRunner):
def min_cluster_node_count(self):
return self.get_datastore_config_property('min_cluster_member_count')
def run_cluster_delete(self, expected_task_name='NONE',
expected_http_code=202):
raise SkipKnownBug(runners.BUG_STOP_DB_IN_CLUSTER)
class VerticaClusterActionsRunner(ClusterActionsRunner):