Initial work to support Xenial

The main goal of this change is to add support for Xenial to
trove/integration and ensure the voting check jobs continue
to pass when switched to Xenial.

- It was decided that Trusty should continue to be supported
to aid in the developer transition to Xenial. Therefore,
the elements have been organized to accomplish this by making
"trusty" and "xenial" specific elements extend the already
existing "ubuntu" elements.
- We tried to do the initial work without impacting the core
Trove code therefore some choices were made - for example,
cluster delete doesn't work for PXC on Xenial (see [1]) so
the test that does this is skipped for PXC.
- Only Mysql, PXC, Postgresql and Redis were fixed for
Xenial and tested as these are the only voting jobs
currently. Future work should sort out the elements for the
rest of the datastores. Note that all of the datastores should
continue to function on Trusty.
- Previously, the "DISTRO" variable was hardcoded to "ubuntu".
This is no longer the case; DISTRO and RELEASE variables will
be automatically set to whatever the base OS is (for example,
if you are running trovestack on Xenial, DISTRO will be set
to "ubuntu" and RELEASE will be set to "xenial". These can
be overriden if needed.

[1] https://bugs.launchpad.net/trove/+bug/1645096

Change-Id: I8e1de6ef31f969ccee88c334a0d5ed03aabd1b51
Depends-On: I3831de12c999ef8818e80ecdb29f1d86ff8cd5c8

integration/scripts/files/elements/ubuntu-mysql/pre-install.d/10-percona-apt-key

@@ -7,7 +7,7 @@ set -e
 set -o xtrace
 
 [ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image"
-[ -n "${RELEASE}" ] || die "RELEASE must be set to either Precise or Quantal"
+[ -n "${RELEASE}" ] || die "RELEASE must be set to a valid Ubuntu release (e.g. trusty)"
 
 # Add Percona GPG key
 mkdir -p /home/${GUEST_USERNAME}/.gnupg

integration/scripts/files/elements/ubuntu-postgresql/pre-install.d/10-postgresql-repo

@@ -3,8 +3,10 @@
 set -e
 set -o xtrace
 
+[ -n "${RELEASE}" ] || die "RELEASE must be set to a valid Ubuntu release (e.g. trusty)"
+
 cat <<EOL > /etc/apt/sources.list.d/postgresql.list
-deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main
+deb http://apt.postgresql.org/pub/repos/apt/ ${RELEASE}-pgdg main
 EOL
 
 wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -

integration/scripts/files/elements/ubuntu-trusty-cassandra/element-deps

@@ -0,0 +1 @@
+ubuntu-cassandra

integration/scripts/files/elements/ubuntu-trusty-couchbase/element-deps

@@ -0,0 +1 @@
+ubuntu-couchbase

integration/scripts/files/elements/ubuntu-trusty-couchdb/element-deps

@@ -0,0 +1 @@
+ubuntu-couchdb

integration/scripts/files/elements/ubuntu-trusty-db2/element-deps

@@ -0,0 +1 @@
+ubuntu-db2

integration/scripts/files/elements/ubuntu-trusty-guest/element-deps

@@ -0,0 +1 @@
+ubuntu-guest

integration/scripts/files/elements/ubuntu-trusty-mariadb/element-deps

@@ -0,0 +1 @@
+ubuntu-mariadb

integration/scripts/files/elements/ubuntu-trusty-mongodb/element-deps

@@ -0,0 +1 @@
+ubuntu-mongodb

integration/scripts/files/elements/ubuntu-trusty-mysql/element-deps

@@ -0,0 +1 @@
+ubuntu-mysql

integration/scripts/files/elements/ubuntu-trusty-percona/element-deps

@@ -0,0 +1 @@
+ubuntu-percona

integration/scripts/files/elements/ubuntu-trusty-postgresql/element-deps

@@ -0,0 +1 @@
+ubuntu-postgresql

integration/scripts/files/elements/ubuntu-trusty-pxc/element-deps

@@ -0,0 +1 @@
+ubuntu-pxc

integration/scripts/files/elements/ubuntu-trusty-redis/element-deps

@@ -0,0 +1 @@
+ubuntu-redis

integration/scripts/files/elements/ubuntu-trusty-vertica/element-deps

@@ -0,0 +1 @@
+ubuntu-vertica

integration/scripts/files/elements/ubuntu-xenial-guest/element-deps

@@ -0,0 +1 @@
+ubuntu-guest

integration/scripts/files/elements/ubuntu-xenial-guest/extra-data.d/20-guest-systemd

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
+# PURPOSE: stages the bootstrap file and upstart conf file while replacing variables so that guest image is properly
+# configured
+
+source $_LIB/die
+
+[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
+
+[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image"
+[ -n "${HOST_SCP_USERNAME}" ] || die "HOST_SCP_USERNAME needs to be set to the user for the host instance"
+[ -n "${CONTROLLER_IP}" ] || die "CONTROLLER_IP needs to be set to the ip address that guests will use to contact the controller"
+[ -n "${ESCAPED_PATH_TROVE}" ] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the trovestack host"
+[ -n "${TROVESTACK_SCRIPTS}" ] || die "TROVESTACK_SCRIPTS needs to be set to the trove/integration/scripts dir"
+[ -n "${ESCAPED_GUEST_LOGDIR}" ] || die "ESCAPED_GUEST_LOGDIR must be set to the escaped guest log dir"
+
+sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/CONTROLLER_IP/${CONTROLLER_IP}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${TROVESTACK_SCRIPTS}/files/trove-guest.systemd.conf > ${TMP_HOOKS_PATH}/trove-guest.service
+

integration/scripts/files/elements/ubuntu-xenial-guest/install.d/20-etc

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: take "staged" trove-guest.conf file and put it in the init directory on guest image
+
+dd if=/tmp/in_target.d/trove-guest.service of=/etc/systemd/system/trove-guest.service
+
+systemctl enable trove-guest.service
+
+

integration/scripts/files/elements/ubuntu-xenial-guest/pre-install.d/01-trim-pkgs

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install basic services and applications
+
+set -e
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y purge acpid\
+                 apport\
+                 apport-symptoms\
+                 apt-transport-https\
+                 aptitude\
+                 at\
+                 bash-completion\
+                 bc\
+                 bind9-host\
+                 bsdmainutils\
+                 busybox-static\
+                 byobu\
+                 command-not-found\
+                 command-not-found-data\
+                 curl\
+                 dbus\
+                 dmidecode\
+                 dosfstools\
+                 ed\
+                 fonts-ubuntu-font-family-console\
+                 friendly-recovery\
+                 ftp\
+                 fuse\
+                 geoip-database\
+                 groff-base\
+                 hdparm\
+                 info\
+                 install-info\
+                 iptables\
+                 iputils-tracepath\
+                 irqbalance\
+                 language-selector-common\
+                 libaccountsservice0\
+                 libevent-2.0-5\
+                 libgeoip1\
+                 libnfnetlink0\
+                 libpcap0.8\
+                 libpci3\
+                 libpipeline1\
+                 libpolkit-gobject-1-0\
+                 libsasl2-modules\
+                 libusb-1.0-0\
+                 lshw\
+                 lsof\
+                 ltrace\
+                 man-db\
+                 mlocate\
+                 mtr-tiny\
+                 nano\
+                 ntfs-3g\
+                 parted\
+                 patch\
+                 plymouth-theme-ubuntu-text\
+                 popularity-contest\
+                 powermgmt-base\
+                 ppp\
+                 screen\
+                 shared-mime-info\
+                 strace\
+                 tcpdump\
+                 telnet\
+                 time\
+                 tmux\
+                 ubuntu-standard\
+                 ufw\
+                 update-manager-core\
+                 update-notifier-common\
+                 usbutils\
+                 uuid-runtime\
+
+# The following packages cannot be removed as they cause cloud-init to be
+# uninstalled in Ubuntu 14.04
+#                 gir1.2-glib-2.0
+#                 libdbus-glib-1-2
+#                 libgirepository-1.0-1
+#                 python-chardet
+#                 python-serial
+#                 xz-utils
+
+apt-get -y autoremove
+

integration/scripts/files/elements/ubuntu-xenial-mysql/element-deps

@@ -0,0 +1 @@
+ubuntu-mysql

integration/scripts/files/elements/ubuntu-xenial-mysql/install.d/30-mysql

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install controller base required packages
+
+set -e
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+
+add-apt-repository 'deb http://archive.ubuntu.com/ubuntu trusty universe'
+apt-get -y  update
+
+apt-get -y install mysql-client-5.6 mysql-server-5.6 percona-xtrabackup${PXB_VERSION_OVERRIDE}
+
+cat >/etc/mysql/conf.d/no_perf_schema.cnf <<_EOF_
+[mysqld]
+performance_schema = off
+_EOF_
+
+mv /etc/mysql/my.cnf.fallback /etc/mysql/my.cnf
+chown mysql:mysql /etc/mysql/my.cnf
+cat >/etc/mysql/my.cnf <<_EOF_
+[mysql]
+
+!includedir /etc/mysql/conf.d/
+_EOF_
+
+rm /etc/init/mysql.conf
+systemctl daemon-reload
+systemctl enable mysql
+

integration/scripts/files/elements/ubuntu-xenial-postgresql/element-deps

@@ -0,0 +1 @@
+ubuntu-postgresql

integration/scripts/files/elements/ubuntu-xenial-postgresql/install.d/11-fix-init-script

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+set -o xtrace
+
+sed -i "s/PIDFile=\/var\/run\/postgresql\/%i.pid/PIDFile=\/var\/run\/postgresql\/postgresql.pid/" /lib/systemd/system/postgresql@.service
+

integration/scripts/files/elements/ubuntu-xenial-pxc/element-deps

@@ -0,0 +1 @@
+ubuntu-pxc

integration/scripts/files/elements/ubuntu-xenial-pxc/install.d/31-fix-my-cnf

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install controller base required packages
+
+set -e
+set -o xtrace
+
+mv /etc/mysql/my.cnf.fallback /etc/mysql/my.cnf
+chown mysql:mysql /etc/mysql/my.cnf
+cat >/etc/mysql/my.cnf <<_EOF_
+[mysql]
+>
+!includedir /etc/mysql/conf.d/
+_EOF_

integration/scripts/files/elements/ubuntu-xenial-redis/element-deps

@@ -0,0 +1 @@
+ubuntu-redis

integration/scripts/files/elements/ubuntu-xenial-redis/install.d/11-fix-init-file

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -e
+set -o xtrace
+
+cat > /lib/systemd/system/redis-server.service << '_EOF_'
+[Unit]
+Description=Advanced key-value store
+After=network.target
+
+[Service]
+Type=forking
+PIDFile=/var/run/redis/redis-server.pid
+User=redis
+Group=redis
+
+Environment=statedir=/var/run/redis
+PermissionsStartOnly=true
+ExecStartPre=/bin/mkdir -p ${statedir}
+ExecStartPre=/bin/chown -R redis:redis ${statedir}
+ExecStart=/usr/bin/redis-server /etc/redis/redis.conf
+ExecReload=/bin/kill -USR2 $MAINPID
+ExecStop=/usr/bin/redis-cli shutdown
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+_EOF_

integration/scripts/functions

@@ -193,30 +193,36 @@ GetOSVersion() {
 function GetDistro() {
     GetOSVersion
     if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) ]]; then
-        # 'Everyone' refers to Ubuntu / Debian releases by the code name adjective
+        DISTRO_NAME=ubuntu
-        DISTRO=$os_CODENAME
+        DISTRO_RELEASE=$os_CODENAME
     elif [[ "$os_VENDOR" =~ (Fedora) ]]; then
-        # For Fedora, just use 'f' and the release
+        DISTRO_NAME=fedora
-        DISTRO="f$os_RELEASE"
+        DISTRO_RELEASE=$os_RELEASE
     elif [[ "$os_VENDOR" =~ (openSUSE) ]]; then
-        DISTRO="opensuse-$os_RELEASE"
+        DISTRO_NAME=opensuse
+        DISTRO_RELEASE=$os_RELEASE
     elif [[ "$os_VENDOR" =~ (SUSE LINUX) ]]; then
+        DISTRO_NAME=sle
         # For SLE, also use the service pack
         if [[ -z "$os_UPDATE" ]]; then
-            DISTRO="sle${os_RELEASE}"
+            DISTRO_RELEASE=$os_RELEASE
         else
-            DISTRO="sle${os_RELEASE}sp${os_UPDATE}"
+            DISTRO_RELEASE="${os_RELEASE}sp${os_UPDATE}"
         fi
     elif [[ "$os_VENDOR" =~ (Red Hat) || "$os_VENDOR" =~ (CentOS) ]]; then
         # Drop the . release as we assume it's compatible
-        DISTRO="rhel${os_RELEASE::1}"
+        DISTRO_NAME=rhel
+        DISTRO_RELEASE=${os_RELEASE::1}
     elif [[ "$os_VENDOR" =~ (XenServer) ]]; then
-        DISTRO="xs$os_RELEASE"
+        DISTRO_NAME=xs
+        DISTRO_RELEASE=$os_RELEASE
     else
         # Catch-all for now is Vendor + Release + Update
-        DISTRO="$os_VENDOR-$os_RELEASE.$os_UPDATE"
+        DISTRO_NAME=$os_VENDOR
+        DISTRO_RELEASE=$os_RELEASE.$os_UPDATE
     fi
-    export DISTRO
+    export DISTRO_NAME
+    export DISTRO_RELEASE
 }
 
 

integration/scripts/functions_qemu

@@ -19,14 +19,8 @@ function build_vm() {
     manage_ssh_keys
 
     if [ $DISTRO == 'ubuntu' ]; then
-        export RELEASE=trusty
         export DIB_RELEASE=$RELEASE
         export DIB_CLOUD_IMAGES=cloud-images.ubuntu.com
-        # Use the apt sources.list on the build host, its almost always preferred
-        if [ -f /etc/apt/sources.list ]; then
-            export DIB_APT_SOURCES=/etc/apt/sources.list
-            EXTRA_ELEMENTS="apt-sources apt-conf-dir"
-        fi
     fi
     if [ $DISTRO == 'fedora' ]; then
         EXTRA_ELEMENTS=selinux-permissive
@@ -54,7 +48,7 @@ function build_vm() {
     local QEMU_IMG_OPTIONS=$(! $(qemu-img | grep -q 'version 1') && echo "--qemu-img-options compat=0.10")
     ${PATH_DISKIMAGEBUILDER}/bin/disk-image-create -a amd64 -o "${VM}" \
         -x ${QEMU_IMG_OPTIONS} ${DISTRO} ${EXTRA_ELEMENTS} vm heat-cfntools \
-        cloud-init-datasources ${DISTRO}-guest ${DISTRO}-${SERVICE_TYPE}
+        cloud-init-datasources ${DISTRO}-${RELEASE}-guest ${DISTRO}-${RELEASE}-${SERVICE_TYPE}
 }
 
 function build_guest_image() {

integration/scripts/trovestack

@@ -48,6 +48,12 @@ function get_default_host_ip() {
 . $TROVESTACK_SCRIPTS/functions
 . $TROVESTACK_SCRIPTS/functions_qemu
 
+# Pre-set DISTRO and RELEASE variables based on host OS
+# Can be overridden by env vars DISTRO and RELEASE
+GetDistro
+export DISTRO=${DISTRO:-$DISTRO_NAME}
+export RELEASE=${RELEASE:-$DISTRO_RELEASE}
+
 # Load global configuration variables.
 . $TROVESTACK_SCRIPTS/trovestack.rc
 . $TROVESTACK_SCRIPTS/reviews.rc
@@ -266,7 +272,8 @@ function install_prep_packages() {
     if is_fedora; then
       pkg_install git gettext
     else
-      pkg_install git-core kvm-ipxe gettext
+      #pkg_install git-core kvm-ipxe gettext
+      pkg_install git-core gettext
     fi
     sudo -H $HTTP_PROXY pip install --upgrade pip dib-utils
 }
@@ -1329,7 +1336,6 @@ function cmd_repl() {
     INT_TEST_OPTIONS=-i cmd_int_tests_white_box --repl --group=_does_not_exist_ $@
 }
 
-
 ###############################################################################
 # Process the user provided command and run the appropriate command
 ###############################################################################

integration/scripts/trovestack.rc

@@ -103,9 +103,6 @@ SWIFT_HASH=12go358snjw24501
 SWIFT_DATA_DIR=${DATA_DIR}/swift
 SWIFT_DISK_IMAGE=${SWIFT_DATA_DIR}/drives/images/swift.img
 
-DISTRO=${DISTRO:-ubuntu}
-#DISTRO=fedora
-
 # The following values can be used to tweak how devstack sets
 # up Trove. If not explicitly set, the defaults in the code are used.
 # To make changes without modifying the repo, add these variables

trove/tests/scenario/runners/__init__.py

@@ -1,2 +1,3 @@
 BUG_EJECT_VALID_MASTER = 1622014
 BUG_WRONG_API_VALIDATION = 1498573
+BUG_STOP_DB_IN_CLUSTER = 1645096

trove/tests/scenario/runners/cluster_actions_runners.py

@@ -22,6 +22,8 @@ import time as timer
 from trove.common import exception
 from trove.common.utils import poll_until
 from trove.tests.scenario.helpers.test_helper import DataType
+from trove.tests.scenario import runners
+from trove.tests.scenario.runners.test_runners import SkipKnownBug
 from trove.tests.scenario.runners.test_runners import TestRunner
 from trove.tests.util.check import TypeCheck
 from troveclient.compat import exceptions
@@ -423,6 +425,10 @@ class PxcClusterActionsRunner(ClusterActionsRunner):
     def min_cluster_node_count(self):
         return self.get_datastore_config_property('min_cluster_member_count')
 
+    def run_cluster_delete(self, expected_task_name='NONE',
+                           expected_http_code=202):
+        raise SkipKnownBug(runners.BUG_STOP_DB_IN_CLUSTER)
+
 
 class VerticaClusterActionsRunner(ClusterActionsRunner):