Fix integration jobs
- Add missing sudos (and remove needless ones) - Don't carry sudo environment, we can add it back if it becomes necessary - Fix permission errors when exporting command output to files - Increase ulimit, we'd run into too many open files in the gate - Add some relevant logfiles to help troubleshoot issues - Re-export path to ensure common file paths are included - Don't assume "ip" is installed, it's provided by iproute - Don't assume "restorecon" is installed, it's provided by policycoreutils - Don't run packstack twice (--allinone will trigger a run even if -gen-answer-file is used, so switch to using --default-password instead) - Use symlink for defaults.yaml instead of common.yaml - Handled Hiera cleanup - Use debug mode in scenarios - Fix tempest subunit retrieval with sudo Co-Authored-By: Martin Magr <mmagr@redhat.com> Co-Authored-By: David Moreau Simard <dms@redhat.com> Change-Id: I7a363308cd9df981c0a1680171b9b2f2b95dd29e
This commit is contained in:
parent
8741eb74ed
commit
f5c884883a
|
@ -24,9 +24,9 @@ controller = Controller()
|
|||
|
||||
PUPPET_DIR = os.path.join(basedefs.DIR_PROJECT_DIR, "puppet")
|
||||
PUPPET_TEMPLATE_DIR = os.path.join(PUPPET_DIR, "templates")
|
||||
HIERA_DEFAULTS_YAML = os.path.join(basedefs.HIERADATA_DIR, "defaults.yaml")
|
||||
# For compatibility with hiera >= 3.0
|
||||
HIERA_COMMON_YAML = os.path.join(basedefs.HIERADATA_DIR, "common.yaml")
|
||||
# For compatibility with hiera < 3.0
|
||||
HIERA_DEFAULTS_YAML = os.path.join(basedefs.HIERADATA_DIR, "defaults.yaml")
|
||||
|
||||
|
||||
class ManifestFiles(object):
|
||||
|
@ -90,11 +90,11 @@ def prependManifestFile(manifest_name, data, marker=''):
|
|||
|
||||
def generateHieraDataFile():
|
||||
os.mkdir(basedefs.HIERADATA_DIR, 0o700)
|
||||
with open(HIERA_DEFAULTS_YAML, 'w') as outfile:
|
||||
with open(HIERA_COMMON_YAML, 'w') as outfile:
|
||||
outfile.write(yaml.dump(controller.CONF,
|
||||
explicit_start=True,
|
||||
default_flow_style=False))
|
||||
os.symlink(HIERA_DEFAULTS_YAML, HIERA_COMMON_YAML)
|
||||
os.symlink(os.path.basename(HIERA_COMMON_YAML), HIERA_DEFAULTS_YAML)
|
||||
|
||||
|
||||
def createFirewallResources(hiera_key, default_value='{}'):
|
||||
|
|
69
run_tests.sh
69
run_tests.sh
|
@ -13,6 +13,8 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
export PATH=$PATH:/usr/local/sbin:/usr/sbin
|
||||
|
||||
SCENARIO=${SCENARIO:-scenario001}
|
||||
|
||||
# We could want to override the default repositories
|
||||
|
@ -24,10 +26,24 @@ DELOREAN_DEPS=${DELOREAN_DEPS:-http://trunk.rdoproject.org/centos7/delorean-deps
|
|||
COPY_LOGS=${COPY_LOGS:-true}
|
||||
|
||||
if [ $(id -u) != 0 ]; then
|
||||
# preserve environment so we can have ZUUL_* params
|
||||
SUDO='sudo -E'
|
||||
SUDO='sudo'
|
||||
|
||||
# Packstack will connect as root to localhost, set-up the keypair and sshd
|
||||
ssh-keygen -t rsa -C "packstack-integration-test" -N "" -f ~/.ssh/id_rsa
|
||||
|
||||
$SUDO mkdir -p /root/.ssh
|
||||
cat ~/.ssh/id_rsa.pub | $SUDO tee -a /root/.ssh/authorized_keys
|
||||
$SUDO chmod 0600 /root/.ssh/authorized_keys
|
||||
$SUDO sed -i 's/^PermitRootLogin no/PermitRootLogin without-password/g' /etc/ssh/sshd_config
|
||||
$SUDO service sshd restart
|
||||
fi
|
||||
|
||||
# Bump ulimit to avoid too many open file errors
|
||||
echo "${USER} soft nofile 65536" | $SUDO tee -a /etc/security/limits.conf
|
||||
echo "${USER} hard nofile 65536" | $SUDO tee -a /etc/security/limits.conf
|
||||
echo "root soft nofile 65536" | $SUDO tee -a /etc/security/limits.conf
|
||||
echo "root hard nofile 65536" | $SUDO tee -a /etc/security/limits.conf
|
||||
|
||||
# Setup repositories
|
||||
if [ "${MANAGE_REPOS}" = true ]; then
|
||||
$SUDO curl ${DELOREAN} -o /etc/yum.repos.d/delorean.repo
|
||||
|
@ -35,7 +51,9 @@ if [ "${MANAGE_REPOS}" = true ]; then
|
|||
fi
|
||||
|
||||
# Install dependencies
|
||||
$SUDO yum -y install yum-plugin-priorities \
|
||||
$SUDO yum -y install puppet \
|
||||
yum-plugin-priorities \
|
||||
iproute \
|
||||
dstat \
|
||||
python-setuptools \
|
||||
openssl-devel \
|
||||
|
@ -45,14 +63,48 @@ $SUDO yum -y install yum-plugin-priorities \
|
|||
libxslt-devel \
|
||||
ruby-devel \
|
||||
openstack-selinux \
|
||||
policycoreutils \
|
||||
"@Development Tools"
|
||||
|
||||
# TO-DO: Packstack should handle Hiera and Puppet configuration, so that it works
|
||||
# no matter the environment
|
||||
$SUDO su -c 'cat > /etc/puppet/puppet.conf <<EOF
|
||||
[main]
|
||||
logdir = /var/log/puppet
|
||||
rundir = /var/run/puppet
|
||||
ssldir = $vardir/ssl
|
||||
hiera_config = /etc/puppet/hiera.yaml
|
||||
|
||||
[agent]
|
||||
classfile = $vardir/classes.txt
|
||||
localconfig = $vardir/localconfig
|
||||
EOF'
|
||||
$SUDO su -c 'cat > /etc/puppet/hiera.yaml <<EOF
|
||||
---
|
||||
:backends:
|
||||
- yaml
|
||||
:yaml:
|
||||
:datadir: /placeholder
|
||||
:hierarchy:
|
||||
- common
|
||||
- defaults
|
||||
- "%{clientcert}"
|
||||
- "%{environment}"
|
||||
- global
|
||||
EOF'
|
||||
|
||||
# To make sure wrong config files are not used
|
||||
if [ -d /home/jenkins/.puppet ]; then
|
||||
$SUDO rm -f /home/jenkins/.puppet
|
||||
fi
|
||||
$SUDO puppet config set hiera_config /etc/puppet/hiera.yaml
|
||||
|
||||
# Setup dstat for resource usage tracing
|
||||
if type "dstat" 2>/dev/null; then
|
||||
$SUDO dstat -tcmndrylpg \
|
||||
--top-cpu-adv \
|
||||
--top-io-adv \
|
||||
--nocolor | $SUDO tee --append /var/log/dstat.log > /dev/null &
|
||||
--nocolor | $SUDO tee -a /var/log/dstat.log > /dev/null &
|
||||
fi
|
||||
|
||||
# Setup packstack
|
||||
|
@ -63,11 +115,12 @@ $SUDO python setup.py install_puppet_modules
|
|||
source ./tests/${SCENARIO}.sh
|
||||
result=$?
|
||||
|
||||
# Generate subunit
|
||||
# Print output and generate subunit if results exist
|
||||
if [ -d /var/lib/tempest ]; then
|
||||
pushd /var/lib/tempest
|
||||
/var/lib/tempest/.venv/bin/testr last --subunit > /var/tmp/packstack/latest/testrepository.subunit || true
|
||||
popd
|
||||
pushd /var/lib/tempest
|
||||
$SUDO /var/lib/tempest/.venv/bin/testr last || true
|
||||
$SUDO bash -c "/var/lib/tempest/.venv/bin/testr last --subunit > /var/tmp/packstack/latest/testrepository.subunit" || true
|
||||
popd
|
||||
fi
|
||||
|
||||
if [ "${COPY_LOGS}" = true ]; then
|
||||
|
|
|
@ -1,9 +1,6 @@
|
|||
#!/bin/bash -ex
|
||||
PACKSTACK_CONFIG_FILE="/tmp/packstack.txt"
|
||||
|
||||
#!/bin/bash
|
||||
if [ $(id -u) != 0 ]; then
|
||||
# preserve environment so we can have ZUUL_* params
|
||||
SUDO='sudo -E'
|
||||
SUDO='sudo'
|
||||
fi
|
||||
|
||||
echo -e "Generating packstack config for:
|
||||
|
@ -22,7 +19,9 @@ echo -e "Generating packstack config for:
|
|||
echo "tempest will run if packstack's installation completes successfully."
|
||||
echo
|
||||
|
||||
packstack --allinone \
|
||||
$SUDO packstack --allinone \
|
||||
--debug \
|
||||
--default-password="packstack" \
|
||||
--os-swift-install=n \
|
||||
--os-horizon-install=n \
|
||||
--os-manila-install=y \
|
||||
|
@ -30,8 +29,4 @@ packstack --allinone \
|
|||
--provision-demo=y \
|
||||
--provision-tempest=y \
|
||||
--run-tempest=y \
|
||||
--run-tempest-tests="smoke TelemetryAlarming" \
|
||||
--gen-answer-file=${PACKSTACK_CONFIG_FILE}
|
||||
sed -i -re "s,(.*_PASSWORD|.*_PW)=.*,\1=packstack," ${PACKSTACK_CONFIG_FILE}
|
||||
|
||||
$SUDO packstack --answer-file=${PACKSTACK_CONFIG_FILE} || export FAILURE="true"
|
||||
--run-tempest-tests="smoke TelemetryAlarming" || export FAILURE=true
|
||||
|
|
|
@ -1,9 +1,6 @@
|
|||
#!/bin/bash -ex
|
||||
PACKSTACK_CONFIG_FILE="/tmp/packstack.txt"
|
||||
|
||||
#!/bin/bash
|
||||
if [ $(id -u) != 0 ]; then
|
||||
# preserve environment so we can have ZUUL_* params
|
||||
SUDO='sudo -E'
|
||||
SUDO='sudo'
|
||||
fi
|
||||
|
||||
echo -e "Generating packstack config for:
|
||||
|
@ -20,7 +17,9 @@ echo -e "Generating packstack config for:
|
|||
echo "tempest will run if packstack's installation completes successfully."
|
||||
echo
|
||||
|
||||
packstack --allinone \
|
||||
$SUDO packstack --allinone \
|
||||
--debug \
|
||||
--default-password="packstack" \
|
||||
--os-aodh-install=n \
|
||||
--os-ceilometer-install=n \
|
||||
--os-gnocchi-install=n \
|
||||
|
@ -33,8 +32,4 @@ packstack --allinone \
|
|||
--provision-demo=y \
|
||||
--provision-tempest=y \
|
||||
--run-tempest=y \
|
||||
--run-tempest-tests="smoke dashboard" \
|
||||
--gen-answer-file=${PACKSTACK_CONFIG_FILE}
|
||||
sed -i -re "s,(.*_PASSWORD|.*_PW)=.*,\1=packstack," ${PACKSTACK_CONFIG_FILE}
|
||||
|
||||
$SUDO packstack --answer-file=${PACKSTACK_CONFIG_FILE} || export FAILURE="true"
|
||||
--run-tempest-tests="smoke dashboard" || export FAILURE=true
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/bin/bash -xe
|
||||
#!/bin/bash
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
@ -14,6 +14,8 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
export PATH=$PATH:/usr/local/sbin:/usr/sbin
|
||||
|
||||
# This script attempts to recover as much generic diagnostic logs as it can
|
||||
LOGROOT=${WORKSPACE:-/tmp}
|
||||
LOGDIR="${LOGROOT}/logs"
|
||||
|
@ -23,7 +25,7 @@ GIT_URL="http://git.openstack.org/cgit"
|
|||
PROJECTS_URL="${GIT_URL}/openstack/governance/plain/reference/projects.yaml"
|
||||
|
||||
if [ $(id -u) != 0 ]; then
|
||||
SUDO='sudo -E'
|
||||
SUDO='sudo'
|
||||
fi
|
||||
$SUDO mkdir -p "${DIAG_LOGDIR}"
|
||||
$SUDO mkdir -p "${CONF_LOGDIR}"
|
||||
|
@ -49,16 +51,17 @@ function get_diag_commands {
|
|||
'yum repolist -v'
|
||||
'rpm -qa'
|
||||
'journalctl --no-pager'
|
||||
'ulimit -n'
|
||||
)
|
||||
|
||||
echo "Installing required RPM packages..."
|
||||
yum -y install coreutils curl file iproute lsof net-tools psmisc
|
||||
$SUDO yum -y install coreutils curl file lsof net-tools psmisc
|
||||
|
||||
echo "Running diagnostic commands..."
|
||||
for ((i = 0; i < ${#commands[@]}; i++)); do
|
||||
# filenames have underscores instead of spaces or slashes
|
||||
filename="$(echo "${commands[$i]}" |sed -e "s%[ \/]%_%g").txt"
|
||||
$SUDO ${commands[$i]} 2>&1 > ${DIAG_LOGDIR}/${filename}
|
||||
$SUDO "${commands[$i]}" 2>&1 | $SUDO tee -a ${DIAG_LOGDIR}/${filename}
|
||||
done
|
||||
}
|
||||
|
||||
|
@ -94,6 +97,11 @@ function get_config_and_logs {
|
|||
'/etc/httpd'
|
||||
'/var/log/httpd'
|
||||
'/var/tmp/packstack'
|
||||
'/var/log/audit'
|
||||
'/var/log/secure'
|
||||
'/var/log/messages'
|
||||
'/etc/puppet/puppet.conf'
|
||||
'/etc/puppet/hiera.yaml'
|
||||
)
|
||||
|
||||
# Add discovered project directories from official governance
|
||||
|
@ -121,20 +129,21 @@ function get_config_and_logs {
|
|||
|
||||
function ensure_log_properties {
|
||||
echo "Making sure directories and files have the right properties..."
|
||||
FIND="${SUDO} find ${LOGDIR} ! -path '*.git/*'"
|
||||
# Ensure files are readable by everyone
|
||||
$FIND -type d -execdir $SUDO chmod 755 '{}' \;
|
||||
$FIND -type f -execdir $SUDO chmod 644 '{}' \;
|
||||
|
||||
# Ensure files are in .txt when possible (web mime type)
|
||||
for file in $(find ${LOGDIR} -type f ! -name "*.txt"); do
|
||||
for file in $($FIND -type f ! -name "*.txt"); do
|
||||
if [[ "$(file --mime-type ${file} |cut -f2 -d :)" =~ text ]]; then
|
||||
$SUDO mv ${file} ${file}.txt
|
||||
fi
|
||||
done
|
||||
|
||||
# Ensure files are readable by everyone
|
||||
$SUDO find $LOGDIR -type d -execdir $SUDO chmod 755 '{}' \;
|
||||
$SUDO find $LOGDIR -type f -execdir $SUDO chmod 644 '{}' \;
|
||||
|
||||
echo "Compressing all text files..."
|
||||
# Compress all files
|
||||
$SUDO find $LOGDIR -iname '*.txt' -execdir gzip -9 {} \+
|
||||
$FIND -iname '*.txt' -execdir gzip -9 {} \+
|
||||
|
||||
echo "Compressed log and configuration can be found in ${LOGDIR}."
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue