Fix integration jobs

- Add missing sudos (and remove needless ones)
- Don't carry sudo environment, we can add it back if it becomes
  necessary
- Fix permission errors when exporting command output to files
- Increase ulimit, we'd run into too many open files in the gate
- Add some relevant logfiles to help troubleshoot issues
- Re-export path to ensure common file paths are included
- Don't assume "ip" is installed, it's provided by iproute
- Don't assume "restorecon" is installed, it's provided by
  policycoreutils
- Don't run packstack twice (--allinone will trigger a run
  even if -gen-answer-file is used, so switch to using
  --default-password instead)
- Use symlink for defaults.yaml instead of common.yaml
- Handled Hiera cleanup
- Use debug mode in scenarios
- Fix tempest subunit retrieval with sudo

Co-Authored-By: Martin Magr <mmagr@redhat.com>
Co-Authored-By: David Moreau Simard <dms@redhat.com>
Change-Id: I7a363308cd9df981c0a1680171b9b2f2b95dd29e

packstack/modules/ospluginutils.py

@@ -24,9 +24,9 @@ controller = Controller()
 
 PUPPET_DIR = os.path.join(basedefs.DIR_PROJECT_DIR, "puppet")
 PUPPET_TEMPLATE_DIR = os.path.join(PUPPET_DIR, "templates")
-HIERA_DEFAULTS_YAML = os.path.join(basedefs.HIERADATA_DIR, "defaults.yaml")
-# For compatibility with hiera >= 3.0
 HIERA_COMMON_YAML = os.path.join(basedefs.HIERADATA_DIR, "common.yaml")
+# For compatibility with hiera < 3.0
+HIERA_DEFAULTS_YAML = os.path.join(basedefs.HIERADATA_DIR, "defaults.yaml")
 
 
 class ManifestFiles(object):
@@ -90,11 +90,11 @@ def prependManifestFile(manifest_name, data, marker=''):
 
 def generateHieraDataFile():
     os.mkdir(basedefs.HIERADATA_DIR, 0o700)
-    with open(HIERA_DEFAULTS_YAML, 'w') as outfile:
+    with open(HIERA_COMMON_YAML, 'w') as outfile:
         outfile.write(yaml.dump(controller.CONF,
                                 explicit_start=True,
                                 default_flow_style=False))
-    os.symlink(HIERA_DEFAULTS_YAML, HIERA_COMMON_YAML)
+    os.symlink(os.path.basename(HIERA_COMMON_YAML), HIERA_DEFAULTS_YAML)
 
 
 def createFirewallResources(hiera_key, default_value='{}'):

run_tests.sh

@@ -13,6 +13,8 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+export PATH=$PATH:/usr/local/sbin:/usr/sbin
+
 SCENARIO=${SCENARIO:-scenario001}
 
 # We could want to override the default repositories
@@ -24,10 +26,24 @@ DELOREAN_DEPS=${DELOREAN_DEPS:-http://trunk.rdoproject.org/centos7/delorean-deps
 COPY_LOGS=${COPY_LOGS:-true}
 
 if [ $(id -u) != 0 ]; then
-    # preserve environment so we can have ZUUL_* params
+    SUDO='sudo'
-    SUDO='sudo -E'
+
+    # Packstack will connect as root to localhost, set-up the keypair and sshd
+    ssh-keygen -t rsa -C "packstack-integration-test" -N "" -f ~/.ssh/id_rsa
+
+    $SUDO mkdir -p /root/.ssh
+    cat ~/.ssh/id_rsa.pub | $SUDO tee -a /root/.ssh/authorized_keys
+    $SUDO chmod 0600 /root/.ssh/authorized_keys
+    $SUDO sed -i 's/^PermitRootLogin no/PermitRootLogin without-password/g' /etc/ssh/sshd_config
+    $SUDO service sshd restart
 fi
 
+# Bump ulimit to avoid too many open file errors
+echo "${USER} soft nofile 65536" | $SUDO tee -a /etc/security/limits.conf
+echo "${USER} hard nofile 65536" | $SUDO tee -a /etc/security/limits.conf
+echo "root soft nofile 65536" | $SUDO tee -a /etc/security/limits.conf
+echo "root hard nofile 65536" | $SUDO tee -a /etc/security/limits.conf
+
 # Setup repositories
 if [ "${MANAGE_REPOS}" = true ]; then
     $SUDO curl ${DELOREAN} -o /etc/yum.repos.d/delorean.repo
@@ -35,7 +51,9 @@ if [ "${MANAGE_REPOS}" = true ]; then
 fi
 
 # Install dependencies
-$SUDO yum -y install yum-plugin-priorities \
+$SUDO yum -y install puppet \
+                     yum-plugin-priorities \
+                     iproute \
                      dstat \
                      python-setuptools \
                      openssl-devel \
@@ -45,14 +63,48 @@ $SUDO yum -y install yum-plugin-priorities \
                      libxslt-devel \
                      ruby-devel \
                      openstack-selinux \
+                     policycoreutils \
                      "@Development Tools"
 
+# TO-DO: Packstack should handle Hiera and Puppet configuration, so that it works
+# no matter the environment
+$SUDO su -c 'cat > /etc/puppet/puppet.conf <<EOF
+[main]
+    logdir = /var/log/puppet
+    rundir = /var/run/puppet
+    ssldir = $vardir/ssl
+    hiera_config = /etc/puppet/hiera.yaml
+
+[agent]
+    classfile = $vardir/classes.txt
+    localconfig = $vardir/localconfig
+EOF'
+$SUDO su -c 'cat > /etc/puppet/hiera.yaml <<EOF
+---
+:backends:
+  - yaml
+:yaml:
+  :datadir: /placeholder
+:hierarchy:
+  - common
+  - defaults
+  - "%{clientcert}"
+  - "%{environment}"
+  - global
+EOF'
+
+# To make sure wrong config files are not used
+if [ -d /home/jenkins/.puppet ]; then
+  $SUDO rm -f /home/jenkins/.puppet
+fi
+$SUDO puppet config set hiera_config /etc/puppet/hiera.yaml
+
 # Setup dstat for resource usage tracing
 if type "dstat" 2>/dev/null; then
   $SUDO dstat -tcmndrylpg \
               --top-cpu-adv \
               --top-io-adv \
-              --nocolor | $SUDO tee --append /var/log/dstat.log > /dev/null &
+              --nocolor | $SUDO tee -a /var/log/dstat.log > /dev/null &
 fi
 
 # Setup packstack
@@ -63,10 +115,11 @@ $SUDO python setup.py install_puppet_modules
 source ./tests/${SCENARIO}.sh
 result=$?
 
-# Generate subunit
+# Print output and generate subunit if results exist
 if [ -d /var/lib/tempest ]; then
     pushd /var/lib/tempest
-  /var/lib/tempest/.venv/bin/testr last --subunit > /var/tmp/packstack/latest/testrepository.subunit || true
+    $SUDO /var/lib/tempest/.venv/bin/testr last || true
+    $SUDO bash -c "/var/lib/tempest/.venv/bin/testr last --subunit > /var/tmp/packstack/latest/testrepository.subunit" || true
     popd
 fi
 

tests/scenario001.sh

@@ -1,9 +1,6 @@
-#!/bin/bash -ex
+#!/bin/bash
-PACKSTACK_CONFIG_FILE="/tmp/packstack.txt"
-
 if [ $(id -u) != 0 ]; then
-    # preserve environment so we can have ZUUL_* params
+    SUDO='sudo'
-    SUDO='sudo -E'
 fi
 
 echo -e "Generating packstack config for:
@@ -22,7 +19,9 @@ echo -e "Generating packstack config for:
 echo "tempest will run if packstack's installation completes successfully."
 echo
 
-packstack --allinone \
+$SUDO packstack --allinone \
+          --debug \
+          --default-password="packstack" \
           --os-swift-install=n \
           --os-horizon-install=n \
           --os-manila-install=y \
@@ -30,8 +29,4 @@ packstack --allinone \
           --provision-demo=y \
           --provision-tempest=y \
           --run-tempest=y \
-          --run-tempest-tests="smoke TelemetryAlarming" \
+          --run-tempest-tests="smoke TelemetryAlarming" || export FAILURE=true
-          --gen-answer-file=${PACKSTACK_CONFIG_FILE}
-sed -i -re "s,(.*_PASSWORD|.*_PW)=.*,\1=packstack," ${PACKSTACK_CONFIG_FILE}
-
-$SUDO packstack --answer-file=${PACKSTACK_CONFIG_FILE} || export FAILURE="true"

tests/scenario002.sh

@@ -1,9 +1,6 @@
-#!/bin/bash -ex
+#!/bin/bash
-PACKSTACK_CONFIG_FILE="/tmp/packstack.txt"
-
 if [ $(id -u) != 0 ]; then
-    # preserve environment so we can have ZUUL_* params
+    SUDO='sudo'
-    SUDO='sudo -E'
 fi
 
 echo -e "Generating packstack config for:
@@ -20,7 +17,9 @@ echo -e "Generating packstack config for:
 echo "tempest will run if packstack's installation completes successfully."
 echo
 
-packstack --allinone \
+$SUDO packstack --allinone \
+          --debug \
+          --default-password="packstack" \
           --os-aodh-install=n \
           --os-ceilometer-install=n \
           --os-gnocchi-install=n \
@@ -33,8 +32,4 @@ packstack --allinone \
           --provision-demo=y \
           --provision-tempest=y \
           --run-tempest=y \
-          --run-tempest-tests="smoke dashboard" \
+          --run-tempest-tests="smoke dashboard" || export FAILURE=true
-          --gen-answer-file=${PACKSTACK_CONFIG_FILE}
-sed -i -re "s,(.*_PASSWORD|.*_PW)=.*,\1=packstack," ${PACKSTACK_CONFIG_FILE}
-
-$SUDO packstack --answer-file=${PACKSTACK_CONFIG_FILE} || export FAILURE="true"

tools/copy-logs.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -xe
+#!/bin/bash
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+export PATH=$PATH:/usr/local/sbin:/usr/sbin
+
 # This script attempts to recover as much generic diagnostic logs as it can
 LOGROOT=${WORKSPACE:-/tmp}
 LOGDIR="${LOGROOT}/logs"
@@ -23,7 +25,7 @@ GIT_URL="http://git.openstack.org/cgit"
 PROJECTS_URL="${GIT_URL}/openstack/governance/plain/reference/projects.yaml"
 
 if [ $(id -u) != 0 ]; then
-    SUDO='sudo -E'
+    SUDO='sudo'
 fi
 $SUDO mkdir -p "${DIAG_LOGDIR}"
 $SUDO mkdir -p "${CONF_LOGDIR}"
@@ -49,16 +51,17 @@ function get_diag_commands {
         'yum repolist -v'
         'rpm -qa'
         'journalctl --no-pager'
+        'ulimit -n'
     )
 
     echo "Installing required RPM packages..."
-    yum -y install coreutils curl file iproute lsof net-tools psmisc
+    $SUDO yum -y install coreutils curl file lsof net-tools psmisc
 
     echo "Running diagnostic commands..."
     for ((i = 0; i < ${#commands[@]}; i++)); do
         # filenames have underscores instead of spaces or slashes
         filename="$(echo "${commands[$i]}" |sed -e "s%[ \/]%_%g").txt"
-        $SUDO ${commands[$i]} 2>&1 > ${DIAG_LOGDIR}/${filename}
+        $SUDO "${commands[$i]}" 2>&1 | $SUDO tee -a ${DIAG_LOGDIR}/${filename}
     done
 }
 
@@ -94,6 +97,11 @@ function get_config_and_logs {
         '/etc/httpd'
         '/var/log/httpd'
         '/var/tmp/packstack'
+        '/var/log/audit'
+        '/var/log/secure'
+        '/var/log/messages'
+        '/etc/puppet/puppet.conf'
+        '/etc/puppet/hiera.yaml'
     )
 
     # Add discovered project directories from official governance
@@ -121,20 +129,21 @@ function get_config_and_logs {
 
 function ensure_log_properties {
     echo "Making sure directories and files have the right properties..."
+    FIND="${SUDO} find ${LOGDIR} ! -path '*.git/*'"
+    # Ensure files are readable by everyone
+    $FIND -type d -execdir $SUDO chmod 755 '{}' \;
+    $FIND -type f -execdir $SUDO chmod 644 '{}' \;
+
     # Ensure files are in .txt when possible (web mime type)
-    for file in $(find ${LOGDIR} -type f ! -name "*.txt"); do
+    for file in $($FIND -type f ! -name "*.txt"); do
         if [[ "$(file --mime-type ${file} |cut -f2 -d :)" =~ text ]]; then
             $SUDO mv ${file} ${file}.txt
         fi
     done
 
-    # Ensure files are readable by everyone
-    $SUDO find $LOGDIR -type d -execdir $SUDO chmod 755 '{}' \;
-    $SUDO find $LOGDIR -type f -execdir $SUDO chmod 644 '{}' \;
-
     echo "Compressing all text files..."
     # Compress all files
-    $SUDO find $LOGDIR -iname '*.txt' -execdir gzip -9 {} \+
+    $FIND -iname '*.txt' -execdir gzip -9 {} \+
 
     echo "Compressed log and configuration can be found in ${LOGDIR}."
 }