Fix integration jobs

- Add missing sudos (and remove needless ones) - Don't carry sudo environment, we can add it back if it becomes necessary - Fix permission errors when exporting command output to files - Increase ulimit, we'd run into too many open files in the gate - Add some relevant logfiles to help troubleshoot issues - Re-export path to ensure common file paths are included - Don't assume "ip" is installed, it's provided by iproute - Don't assume "restorecon" is installed, it's provided by policycoreutils - Don't run packstack twice (--allinone will trigger a run even if -gen-answer-file is used, so switch to using --default-password instead) - Use symlink for defaults.yaml instead of common.yaml - Handled Hiera cleanup - Use debug mode in scenarios - Fix tempest subunit retrieval with sudo Co-Authored-By: Martin Magr <mmagr@redhat.com> Co-Authored-By: David Moreau Simard <dms@redhat.com> Change-Id: I7a363308cd9df981c0a1680171b9b2f2b95dd29e
2016-01-30 10:10:32 -05:00
parent 8741eb74ed
commit f5c884883a
5 changed files with 96 additions and 44 deletions
--- a/packstack/modules/ospluginutils.py
+++ b/packstack/modules/ospluginutils.py
@@ -24,9 +24,9 @@ controller = Controller()

 PUPPET_DIR = os.path.join(basedefs.DIR_PROJECT_DIR, "puppet")
 PUPPET_TEMPLATE_DIR = os.path.join(PUPPET_DIR, "templates")
-HIERA_DEFAULTS_YAML = os.path.join(basedefs.HIERADATA_DIR, "defaults.yaml")
-# For compatibility with hiera >= 3.0
 HIERA_COMMON_YAML = os.path.join(basedefs.HIERADATA_DIR, "common.yaml")
+# For compatibility with hiera < 3.0
+HIERA_DEFAULTS_YAML = os.path.join(basedefs.HIERADATA_DIR, "defaults.yaml")


 class ManifestFiles(object):
@@ -90,11 +90,11 @@ def prependManifestFile(manifest_name, data, marker=''):

 def generateHieraDataFile():
    os.mkdir(basedefs.HIERADATA_DIR, 0o700)
-    with open(HIERA_DEFAULTS_YAML, 'w') as outfile:
+    with open(HIERA_COMMON_YAML, 'w') as outfile:
        outfile.write(yaml.dump(controller.CONF,
                                explicit_start=True,
                                default_flow_style=False))
-    os.symlink(HIERA_DEFAULTS_YAML, HIERA_COMMON_YAML)
+    os.symlink(os.path.basename(HIERA_COMMON_YAML), HIERA_DEFAULTS_YAML)


 def createFirewallResources(hiera_key, default_value='{}'):
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -13,6 +13,8 @@
 # License for the specific language governing permissions and limitations
 # under the License.

+export PATH=$PATH:/usr/local/sbin:/usr/sbin
+
 SCENARIO=${SCENARIO:-scenario001}

 # We could want to override the default repositories
@@ -24,10 +26,24 @@ DELOREAN_DEPS=${DELOREAN_DEPS:-http://trunk.rdoproject.org/centos7/delorean-deps
 COPY_LOGS=${COPY_LOGS:-true}

 if [ $(id -u) != 0 ]; then
-    # preserve environment so we can have ZUUL_* params
-    SUDO='sudo -E'
+    SUDO='sudo'
+
+    # Packstack will connect as root to localhost, set-up the keypair and sshd
+    ssh-keygen -t rsa -C "packstack-integration-test" -N "" -f ~/.ssh/id_rsa
+
+    $SUDO mkdir -p /root/.ssh
+    cat ~/.ssh/id_rsa.pub | $SUDO tee -a /root/.ssh/authorized_keys
+    $SUDO chmod 0600 /root/.ssh/authorized_keys
+    $SUDO sed -i 's/^PermitRootLogin no/PermitRootLogin without-password/g' /etc/ssh/sshd_config
+    $SUDO service sshd restart
 fi

+# Bump ulimit to avoid too many open file errors
+echo "${USER} soft nofile 65536" | $SUDO tee -a /etc/security/limits.conf
+echo "${USER} hard nofile 65536" | $SUDO tee -a /etc/security/limits.conf
+echo "root soft nofile 65536" | $SUDO tee -a /etc/security/limits.conf
+echo "root hard nofile 65536" | $SUDO tee -a /etc/security/limits.conf
+
 # Setup repositories
 if [ "${MANAGE_REPOS}" = true ]; then
    $SUDO curl ${DELOREAN} -o /etc/yum.repos.d/delorean.repo
@@ -35,7 +51,9 @@ if [ "${MANAGE_REPOS}" = true ]; then
 fi

 # Install dependencies
-$SUDO yum -y install yum-plugin-priorities \
+$SUDO yum -y install puppet \
+                     yum-plugin-priorities \
+                     iproute \
                     dstat \
                     python-setuptools \
                     openssl-devel \
@@ -45,14 +63,48 @@ $SUDO yum -y install yum-plugin-priorities \
                     libxslt-devel \
                     ruby-devel \
                     openstack-selinux \
+                     policycoreutils \
                     "@Development Tools"

+# TO-DO: Packstack should handle Hiera and Puppet configuration, so that it works
+# no matter the environment
+$SUDO su -c 'cat > /etc/puppet/puppet.conf <<EOF
+[main]
+    logdir = /var/log/puppet
+    rundir = /var/run/puppet
+    ssldir = $vardir/ssl
+    hiera_config = /etc/puppet/hiera.yaml
+
+[agent]
+    classfile = $vardir/classes.txt
+    localconfig = $vardir/localconfig
+EOF'
+$SUDO su -c 'cat > /etc/puppet/hiera.yaml <<EOF
+---
+:backends:
+  - yaml
+:yaml:
+  :datadir: /placeholder
+:hierarchy:
+  - common
+  - defaults
+  - "%{clientcert}"
+  - "%{environment}"
+  - global
+EOF'
+
+# To make sure wrong config files are not used
+if [ -d /home/jenkins/.puppet ]; then
+  $SUDO rm -f /home/jenkins/.puppet
+fi
+$SUDO puppet config set hiera_config /etc/puppet/hiera.yaml
+
 # Setup dstat for resource usage tracing
 if type "dstat" 2>/dev/null; then
  $SUDO dstat -tcmndrylpg \
              --top-cpu-adv \
              --top-io-adv \
-              --nocolor | $SUDO tee --append /var/log/dstat.log > /dev/null &
+              --nocolor | $SUDO tee -a /var/log/dstat.log > /dev/null &
 fi

 # Setup packstack
@@ -63,11 +115,12 @@ $SUDO python setup.py install_puppet_modules
 source ./tests/${SCENARIO}.sh
 result=$?

-# Generate subunit
+# Print output and generate subunit if results exist
 if [ -d /var/lib/tempest ]; then
-  pushd /var/lib/tempest
-  /var/lib/tempest/.venv/bin/testr last --subunit > /var/tmp/packstack/latest/testrepository.subunit || true
-  popd
+    pushd /var/lib/tempest
+    $SUDO /var/lib/tempest/.venv/bin/testr last || true
+    $SUDO bash -c "/var/lib/tempest/.venv/bin/testr last --subunit > /var/tmp/packstack/latest/testrepository.subunit" || true
+    popd
 fi

 if [ "${COPY_LOGS}" = true ]; then
--- a/tests/scenario001.sh
+++ b/tests/scenario001.sh
@@ -1,9 +1,6 @@
-#!/bin/bash -ex
-PACKSTACK_CONFIG_FILE="/tmp/packstack.txt"
-
+#!/bin/bash
 if [ $(id -u) != 0 ]; then
-    # preserve environment so we can have ZUUL_* params
-    SUDO='sudo -E'
+    SUDO='sudo'
 fi

 echo -e "Generating packstack config for:
@@ -22,7 +19,9 @@ echo -e "Generating packstack config for:
 echo "tempest will run if packstack's installation completes successfully."
 echo

-packstack --allinone \
+$SUDO packstack --allinone \
+          --debug \
+          --default-password="packstack" \
          --os-swift-install=n \
          --os-horizon-install=n \
          --os-manila-install=y \
@@ -30,8 +29,4 @@ packstack --allinone \
          --provision-demo=y \
          --provision-tempest=y \
          --run-tempest=y \
-          --run-tempest-tests="smoke TelemetryAlarming" \
-          --gen-answer-file=${PACKSTACK_CONFIG_FILE}
-sed -i -re "s,(.*_PASSWORD|.*_PW)=.*,\1=packstack," ${PACKSTACK_CONFIG_FILE}
-
-$SUDO packstack --answer-file=${PACKSTACK_CONFIG_FILE} || export FAILURE="true"
+          --run-tempest-tests="smoke TelemetryAlarming" || export FAILURE=true
--- a/tests/scenario002.sh
+++ b/tests/scenario002.sh
@@ -1,9 +1,6 @@
-#!/bin/bash -ex
-PACKSTACK_CONFIG_FILE="/tmp/packstack.txt"
-
+#!/bin/bash
 if [ $(id -u) != 0 ]; then
-    # preserve environment so we can have ZUUL_* params
-    SUDO='sudo -E'
+    SUDO='sudo'
 fi

 echo -e "Generating packstack config for:
@@ -20,7 +17,9 @@ echo -e "Generating packstack config for:
 echo "tempest will run if packstack's installation completes successfully."
 echo

-packstack --allinone \
+$SUDO packstack --allinone \
+          --debug \
+          --default-password="packstack" \
          --os-aodh-install=n \
          --os-ceilometer-install=n \
          --os-gnocchi-install=n \
@@ -33,8 +32,4 @@ packstack --allinone \
          --provision-demo=y \
          --provision-tempest=y \
          --run-tempest=y \
-          --run-tempest-tests="smoke dashboard" \
-          --gen-answer-file=${PACKSTACK_CONFIG_FILE}
-sed -i -re "s,(.*_PASSWORD|.*_PW)=.*,\1=packstack," ${PACKSTACK_CONFIG_FILE}
-
-$SUDO packstack --answer-file=${PACKSTACK_CONFIG_FILE} || export FAILURE="true"
+          --run-tempest-tests="smoke dashboard" || export FAILURE=true
--- a/tools/copy-logs.sh
+++ b/tools/copy-logs.sh
@@ -1,4 +1,4 @@
-#!/bin/bash -xe
+#!/bin/bash
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+export PATH=$PATH:/usr/local/sbin:/usr/sbin
+
 # This script attempts to recover as much generic diagnostic logs as it can
 LOGROOT=${WORKSPACE:-/tmp}
 LOGDIR="${LOGROOT}/logs"
@@ -23,7 +25,7 @@ GIT_URL="http://git.openstack.org/cgit"
 PROJECTS_URL="${GIT_URL}/openstack/governance/plain/reference/projects.yaml"

 if [ $(id -u) != 0 ]; then
-    SUDO='sudo -E'
+    SUDO='sudo'
 fi
 $SUDO mkdir -p "${DIAG_LOGDIR}"
 $SUDO mkdir -p "${CONF_LOGDIR}"
@@ -49,16 +51,17 @@ function get_diag_commands {
        'yum repolist -v'
        'rpm -qa'
        'journalctl --no-pager'
+        'ulimit -n'
    )

    echo "Installing required RPM packages..."
-    yum -y install coreutils curl file iproute lsof net-tools psmisc
+    $SUDO yum -y install coreutils curl file lsof net-tools psmisc

    echo "Running diagnostic commands..."
    for ((i = 0; i < ${#commands[@]}; i++)); do
        # filenames have underscores instead of spaces or slashes
        filename="$(echo "${commands[$i]}" |sed -e "s%[ \/]%_%g").txt"
-        $SUDO ${commands[$i]} 2>&1 > ${DIAG_LOGDIR}/${filename}
+        $SUDO "${commands[$i]}" 2>&1 | $SUDO tee -a ${DIAG_LOGDIR}/${filename}
    done
 }

@@ -94,6 +97,11 @@ function get_config_and_logs {
        '/etc/httpd'
        '/var/log/httpd'
        '/var/tmp/packstack'
+        '/var/log/audit'
+        '/var/log/secure'
+        '/var/log/messages'
+        '/etc/puppet/puppet.conf'
+        '/etc/puppet/hiera.yaml'
    )

    # Add discovered project directories from official governance
@@ -121,20 +129,21 @@ function get_config_and_logs {

 function ensure_log_properties {
    echo "Making sure directories and files have the right properties..."
+    FIND="${SUDO} find ${LOGDIR} ! -path '*.git/*'"
+    # Ensure files are readable by everyone
+    $FIND -type d -execdir $SUDO chmod 755 '{}' \;
+    $FIND -type f -execdir $SUDO chmod 644 '{}' \;
+
    # Ensure files are in .txt when possible (web mime type)
-    for file in $(find ${LOGDIR} -type f ! -name "*.txt"); do
+    for file in $($FIND -type f ! -name "*.txt"); do
        if [[ "$(file --mime-type ${file} |cut -f2 -d :)" =~ text ]]; then
            $SUDO mv ${file} ${file}.txt
        fi
    done

-    # Ensure files are readable by everyone
-    $SUDO find $LOGDIR -type d -execdir $SUDO chmod 755 '{}' \;
-    $SUDO find $LOGDIR -type f -execdir $SUDO chmod 644 '{}' \;
-
    echo "Compressing all text files..."
    # Compress all files
-    $SUDO find $LOGDIR -iname '*.txt' -execdir gzip -9 {} \+
+    $FIND -iname '*.txt' -execdir gzip -9 {} \+

    echo "Compressed log and configuration can be found in ${LOGDIR}."
 }