Add OOOQ multinode test support
This commit adds support to run nodepool multinode tests using tripleo-quickstart as the runner. Change-Id: I11827893c716faf2d295f86ac747470e45613505 Depends-On: I0688d7538d4c6fe4d24d5152ee4c8dae2557508f
This commit is contained in:
Ben Kero
John Trowbridge
parent
a2580615dd
commit
9e391e8a0f
|
|
@ -0,0 +1,34 @@
|
|||
---
|
||||
- include: quickstart.yml
|
||||
|
||||
- name: Prepare the undercloud for installation
|
||||
hosts: undercloud
|
||||
roles:
|
||||
- undercloud-setup
|
||||
tags:
|
||||
- undercloud-setup
|
||||
environment:
|
||||
WORKSPACE: "{{ lookup('env','WORKSPACE') }}"
|
||||
SSH_OPTIONS: "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=Verbose -o PasswordAuthentication=no -o ConnectionAttempts=32 -l jenkins"
|
||||
|
||||
- name: Install the undercloud
|
||||
hosts: undercloud
|
||||
roles:
|
||||
- undercloud-deploy
|
||||
|
||||
- name: Validate the undercloud
|
||||
hosts: undercloud
|
||||
roles:
|
||||
- validate-undercloud
|
||||
|
||||
- name: Run overcloud prep roles
|
||||
hosts: undercloud
|
||||
gather_facts: no
|
||||
roles:
|
||||
- { role: overcloud-prep-images, step_overcloud_image: false, step_glance_upload: true, step_register: false }
|
||||
|
||||
- name: Deploy the overcloud
|
||||
hosts: undercloud
|
||||
gather_facts: no
|
||||
roles:
|
||||
- overcloud-deploy
|
||||
|
|
@ -0,0 +1,141 @@
|
|||
extra_args: '-e /usr/share/openstack-tripleo-heat-templates/environments/deployed-server-environment.yaml \
|
||||
-e ${TRIPLEO_ROOT}/tripleo-ci/test-environments/multinode.yaml \
|
||||
-e ${TRIPLEO_ROOT}/tripleo-ci/test-environments/worker-config.yaml \
|
||||
--compute-scale 0'
|
||||
ssl_overcloud: false
|
||||
network_isolation: false
|
||||
flavor_args: ""
|
||||
undercloud_local_interface: "br-ex"
|
||||
undercloud_generate_service_certificate: false
|
||||
undercloud_check_idempotency: true
|
||||
undercloud_check_sanity: true
|
||||
|
||||
toci_vxlan_networking: true
|
||||
toci_vxlan_networking_multinode: true
|
||||
hostname_correction: true
|
||||
package_installs: true
|
||||
non_root_user_setup: true
|
||||
|
||||
step_overcloud_image: false
|
||||
step_glance_upload: true
|
||||
step_register: false
|
||||
|
||||
hypervisor_wait: false
|
||||
deployed_server: true
|
||||
validation_args: "--validation-errors-nonfatal"
|
||||
deploy_timeout: 30
|
||||
|
||||
undercloud_conf_extras: "net_config_override = /opt/stack/new/tripleo-ci/undercloud-configs/net-config-multinode.json.template"
|
||||
|
||||
enable_vbmc: false
|
||||
|
||||
# TODO(trown): Create repo configs for each release in the release configs
|
||||
repos:
|
||||
- type: generic
|
||||
reponame: delorean
|
||||
filename: delorean.repo
|
||||
baseurl: http://buildlogs.centos.org/centos/7/cloud/x86_64/rdo-trunk-master-tripleo/
|
||||
hash_url: http://buildlogs.centos.org/centos/7/cloud/x86_64/rdo-trunk-master-tripleo/delorean.repo
|
||||
priority: 20
|
||||
|
||||
- type: generic
|
||||
reponame: delorean-current
|
||||
filename: delorean-current.repo
|
||||
baseurl: http://trunk.rdoproject.org/centos7/current
|
||||
hash_url: http://trunk.rdoproject.org/centos7/current/delorean.repo
|
||||
priority: 10
|
||||
includepkgs:
|
||||
- diskimage-builder
|
||||
- instack
|
||||
- instack-undercloud
|
||||
- os-apply-config
|
||||
- os-cloud-config
|
||||
- os-collect-config
|
||||
- os-net-config
|
||||
- os-refresh-config
|
||||
- python-tripleoclient
|
||||
- openstack-tripleo-common
|
||||
- openstack-tripleo-heat-templates
|
||||
- openstack-tripleo-image-elements
|
||||
- openstack-tripleo
|
||||
- openstack-tripleo-puppet-elements
|
||||
- openstack-puppet-modules
|
||||
- openstack-tripleo-ui
|
||||
- puppet-*
|
||||
|
||||
- type: file
|
||||
filename: delorean-deps.repo
|
||||
down_url: http://trunk.rdoproject.org/centos7-master/delorean-deps.repo
|
||||
|
||||
- type: package
|
||||
pkg_name: centos-release-ceph-jewel
|
||||
custom_cmd: 'yum install -y --enablerepo=extras'
|
||||
|
||||
artcl_publish: false
|
||||
artcl_rsync_logs: false
|
||||
artcl_collect_list:
|
||||
- /var/log/
|
||||
- /var/lib/mysql
|
||||
- /home/jenkins/.instack/install-undercloud.log
|
||||
- /home/jenkins/stackrc
|
||||
- /home/jenkins/overcloudrc
|
||||
- /home/jenkins/*.log
|
||||
- /home/jenkins/*.txt
|
||||
- /home/jenkins/*.json
|
||||
- /home/jenkins/*.conf
|
||||
- /home/jenkins/*.yml
|
||||
- /home/jenkins/*.yaml
|
||||
- /home/jenkins/*.sh
|
||||
- /home/jenkins/deploy-overcloudrc
|
||||
- /home/jenkins/network-environment.yaml
|
||||
- /home/jenkins/tempest/*.xml
|
||||
- /home/jenkins/tempest/*.html
|
||||
- /home/jenkins/tempest/*.log
|
||||
- /home/jenkins/tempest/*.sh
|
||||
- /home/jenkins/tempest/etc/*.conf
|
||||
- /home/jenkins/local_tht
|
||||
- /usr/share/openstack-tripleo-heat-templates
|
||||
- /tmp/*.yml
|
||||
- /tmp/*.yaml
|
||||
- /etc/aodh
|
||||
- /etc/ceilometer
|
||||
- /etc/cinder
|
||||
- /etc/dnsmasq-ironic.conf
|
||||
- /etc/docker-registry.yml
|
||||
- /etc/glance
|
||||
- /etc/gnocchi
|
||||
- /etc/heat
|
||||
- /etc/haproxy
|
||||
- /etc/httpd
|
||||
- /etc/ironic
|
||||
- /etc/ironic-inspector
|
||||
- /etc/keepalived
|
||||
- /etc/keystone
|
||||
- /etc/libvirt
|
||||
- /etc/manila
|
||||
- /etc/mistral
|
||||
- /etc/mongod.conf
|
||||
- /etc/mongos.conf
|
||||
- /etc/neutron
|
||||
- /etc/nova
|
||||
- /etc/ntp
|
||||
- /etc/ntp.conf
|
||||
- /etc/os-collect-config.conf
|
||||
- /etc/os-net-config
|
||||
- /etc/puppet
|
||||
- /etc/qpid
|
||||
- /etc/qpidd.conf
|
||||
- /etc/rabbitmq
|
||||
- /etc/redis
|
||||
- /etc/redis.conf*
|
||||
- /etc/resolv.conf
|
||||
- /etc/sahara
|
||||
- /etc/selinux
|
||||
- /etc/swift
|
||||
- /etc/sysconfig
|
||||
- /etc/systemd
|
||||
- /etc/trove
|
||||
- /etc/yum.repos.d
|
||||
- /etc/zaqar
|
||||
artcl_gzip_only: true
|
||||
|
||||
|
|
@ -105,14 +105,6 @@ elif [[ "$TOCI_JOBTYPE" =~ "periodic" && "$TOCI_JOBTYPE" =~ "-nonha" ]]; then
|
|||
UNDERCLOUD_IDEMPOTENT=1
|
||||
fi
|
||||
|
||||
# TODO(trown) remove this when we get the multinode-oooq job[1] in a reasonable
|
||||
# state. Right now it runs on every quickstart and quickstart-extras patch
|
||||
# wasting resources.
|
||||
# [1] https://review.openstack.org/#/c/416110
|
||||
if [[ $TOCI_JOBTYPE =~ multinode-oooq ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# start dstat early
|
||||
# TODO add it to the gate image building
|
||||
sudo yum install -y dstat nmap-ncat #nc is for metrics
|
||||
|
|
@ -252,7 +244,11 @@ for JOB_TYPE_PART in $(sed 's/-/ /g' <<< "${TOCI_JOBTYPE:-}") ; do
|
|||
;;
|
||||
oooq)
|
||||
export OOOQ=1
|
||||
TOCIRUNNER="./toci_instack_oooq.sh"
|
||||
if [[ "$TOCI_JOBTYPE" =~ "multinode" ]]; then
|
||||
TOCIRUNNER="./toci_instack_oooq_multinode.sh"
|
||||
else
|
||||
TOCIRUNNER="./toci_instack_oooq.sh"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
|
|
|||
|
|
@ -0,0 +1,172 @@
|
|||
#!/usr/bin/env bash
|
||||
set -eux
|
||||
set -o pipefail
|
||||
|
||||
## Signal to toci_gate_test.sh we've started by
|
||||
touch /tmp/toci.started
|
||||
|
||||
exit_value=0
|
||||
export CURRENT_DIR=$(dirname ${BASH_SOURCE[0]:-$0})
|
||||
export TRIPLEO_CI_DIR=$CURRENT_DIR/../
|
||||
|
||||
# TODO(sshnaidm): remove this immediately when settings are in yaml files
|
||||
source $TRIPLEO_CI_DIR/tripleo-ci/scripts/common_vars.bash
|
||||
source $TRIPLEO_CI_DIR/tripleo-ci/scripts/common_functions.sh
|
||||
#source $TRIPLEO_CI_DIR/tripleo-ci/scripts/metrics.bash
|
||||
|
||||
# Clear out any puppet modules on the node placed their by infra configuration
|
||||
sudo rm -rf /etc/puppet/modules/*
|
||||
|
||||
# Copy nodepool public key to jenkins user
|
||||
sudo cp /etc/nodepool/id_rsa.pub $HOME/.ssh/
|
||||
sudo chown $USER:$USER $HOME/.ssh/id_rsa.pub
|
||||
|
||||
export USE_DELOREAN=0
|
||||
# TODO(sshnaidm): Hack for gate role, the excluded repos should be handled
|
||||
# properly there. To fix ansible gate role.
|
||||
export ZUUL_CHANGES=${ZUUL_CHANGES:-}
|
||||
export ZUUL_CHANGES=$(echo $ZUUL_CHANGES | python -c 'import sys; print "^".join([i for i in sys.stdin.readline().split("^") if "openstack-infra/tripleo-ci:" not in i])')
|
||||
if [[ -n "$ZUUL_CHANGES" ]]; then
|
||||
export ZUUL_HOST="review.openstack.org"
|
||||
export USE_DELOREAN=1
|
||||
fi
|
||||
|
||||
# TODO(sshnaidm): To create tripleo-ci special yaml config files in oooq
|
||||
# for every TOCI_JOBTYPE, i.e. ovb-nonha-ipv6.yml
|
||||
if [[ "$TOCI_JOBTYPE" =~ "-ha" ]]; then
|
||||
CONFIG=${CONFIG:-"$TRIPLEO_ROOT/tripleo-quickstart/config/general_config/ha.yml"}
|
||||
elif [[ "$TOCI_JOBTYPE" =~ "-nonha" ]]; then
|
||||
CONFIG=${CONFIG:-"$TRIPLEO_ROOT/tripleo-quickstart/config/general_config/minimal.yml"}
|
||||
else
|
||||
CONFIG=${CONFIG:-"$TRIPLEO_ROOT/tripleo-quickstart/config/general_config/minimal.yml"}
|
||||
fi
|
||||
|
||||
# Add jenkin user's SSH key to root authorized_keys for Ansible to run
|
||||
sudo mkdir -p /root/.ssh/
|
||||
sudo cp ${HOME}/.ssh/authorized_keys /root/.ssh/
|
||||
sudo chmod 0600 /root/.ssh/authorized_keys
|
||||
sudo chown root:root /root/.ssh/authorized_keys
|
||||
|
||||
# TODO(bkero): Use an ansible role to create this file
|
||||
sudo yum install -y qemu-img
|
||||
qemu-img create -f qcow2 /home/jenkins/overcloud-full.qcow2 1G
|
||||
|
||||
# Bootstrap the subnodes
|
||||
$TRIPLEO_CI_DIR/tripleo-ci/scripts/tripleo.sh --bootstrap-subnodes 2>&1 | sudo dd of=/var/log/bootstrap-subnodes.log || (tail -n 50 /var/log/bootstrap-subnodes.log && false)
|
||||
|
||||
# Install our test cert so SSL tests work
|
||||
sudo cp $TRIPLEO_ROOT/tripleo-ci/test-environments/overcloud-cacert.pem /etc/pki/ca-trust/source/anchors/
|
||||
sudo cp $TRIPLEO_ROOT/tripleo-ci/test-environments/overcloud-cacert-ipv6.pem /etc/pki/ca-trust/source/anchors/
|
||||
sudo update-ca-trust extract
|
||||
|
||||
# TODO(sshnaidm): to move these variables to jobs yaml config files (see above)
|
||||
if [[ "${STABLE_RELEASE}" =~ ^(liberty|mitaka)$ ]] ; then
|
||||
export OVERCLOUD_DEPLOY_ARGS="$OVERCLOUD_DEPLOY_ARGS -e $TRIPLEO_ROOT/tripleo-ci/test-environments/worker-config-mitaka-and-below.yaml"
|
||||
else
|
||||
export OVERCLOUD_DEPLOY_ARGS="$OVERCLOUD_DEPLOY_ARGS -e $TRIPLEO_ROOT/tripleo-ci/test-environments/worker-config.yaml -e /usr/share/openstack-tripleo-heat-templates/environments/low-memory-usage.yaml"
|
||||
fi
|
||||
export OPT_WORKDIR=${WORKSPACE}/.quickstart
|
||||
export OOOQ_LOGS=/var/log/oooq
|
||||
export OOO_WORKDIR_LOCAL=$HOME
|
||||
export OOOQ_DEFAULT_ARGS=" --working-dir $OPT_WORKDIR --retain-inventory -T none -e working_dir=$OOO_WORKDIR_LOCAL -R ${STABLE_RELEASE:-master}"
|
||||
|
||||
export OOOQ_ARGS=" --working-dir ${OPT_WORKDIR} \
|
||||
-e working_dir=${OOO_WORKDIR_LOCAL} \
|
||||
--bootstrap \
|
||||
--no-clone \
|
||||
--retain-inventory \
|
||||
--tags undercloud-setup,undercloud-scripts,undercloud-install,undercloud-post-install,overcloud-scripts,overcloud-deploy \
|
||||
--teardown none \
|
||||
--release ${STABLE_RELEASE:-master} \
|
||||
--config config/general_config/minimal.yml \
|
||||
--extra-vars @${TRIPLEO_ROOT}/tripleo-ci/scripts/quickstart/multinode-settings.yml \
|
||||
--playbook multinode-playbook.yml \
|
||||
--requirements requirements.txt \
|
||||
--requirements quickstart-extras-requirements.txt \
|
||||
127.0.0.2"
|
||||
|
||||
#shopt -s extglob
|
||||
#rm -rf /opt/stack/new/!(tripleo-ci|tripleo-quickstart|tripleo-quickstart-extras)
|
||||
# End of cleaning
|
||||
# HINT: If there's no enough space, remove swap file in /root/
|
||||
|
||||
# TODO(sshnaidm): when collect-logs role will have the same functionality,
|
||||
# replace postci function with this role (see in the end of file).
|
||||
trap "exit_val=\$?; [ \$exit_val != 0 ] && echo ERROR DURING PREVIOUS COMMAND ^^^ && echo 'See postci.txt in the logs directory for debugging details'; postci \$exit_val 2>&1 | ts '%Y-%m-%d %H:%M:%S.000 |' > $WORKSPACE/logs/postci.log 2>&1" EXIT
|
||||
|
||||
mkdir -p $WORKSPACE/logs
|
||||
|
||||
export IP_DEVICE=${IP_DEVICE:-"eth0"}
|
||||
MY_IP=$(ip addr show dev $IP_DEVICE | awk '/inet / {gsub("/.*", "") ; print $2}')
|
||||
MY_IP_eth1=$(ip addr show dev eth1 | awk '/inet / {gsub("/.*", "") ; print $2}') || MY_IP_eth1=""
|
||||
|
||||
export http_proxy=""
|
||||
undercloud_net_range="192.168.24."
|
||||
undercloud_services_ip=$undercloud_net_range"1"
|
||||
undercloud_haproxy_public_ip=$undercloud_net_range"2"
|
||||
undercloud_haproxy_admin_ip=$undercloud_net_range"3"
|
||||
export no_proxy=$undercloud_services_ip,$undercloud_haproxy_public_ip,$undercloud_haproxy_admin_ip,$MY_IP,$MY_IP_eth1
|
||||
|
||||
[[ ! -e $OPT_WORKDIR ]] && mkdir -p $OPT_WORKDIR && sudo chown -R ${USER} $OPT_WORKDIR
|
||||
sudo mkdir $OOOQ_LOGS && sudo chown -R ${USER} $OOOQ_LOGS
|
||||
|
||||
# make the requirements point to local checkout of tripleo-quickstart-extras
|
||||
echo "file://${TRIPLEO_ROOT}/tripleo-quickstart-extras/#egg=tripleo-quickstart-extras" > ${TRIPLEO_ROOT}/tripleo-quickstart/quickstart-extras-requirements.txt
|
||||
|
||||
# TODO(bkero): Correct this with a dynamic inventory configuration
|
||||
sed -i 's/^undercloud ansible_host=undercloud/undercloud ansible_host=127.0.0.2/' $TRIPLEO_ROOT/tripleo-ci/scripts/hosts
|
||||
|
||||
cp $TRIPLEO_ROOT/tripleo-ci/scripts/hosts $OPT_WORKDIR/hosts
|
||||
cp $TRIPLEO_ROOT/tripleo-ci/scripts/quickstart/*yml $TRIPLEO_ROOT/tripleo-quickstart/playbooks/
|
||||
$TRIPLEO_ROOT/tripleo-quickstart/quickstart.sh --install-deps
|
||||
|
||||
pushd $TRIPLEO_ROOT/tripleo-quickstart/
|
||||
# TODO(sshnaidm): fix inventory role with prepares ssh.config.ansible,
|
||||
# it's not usable here right now. Hopefully ssh config is not required for us.
|
||||
export ANSIBLE_SSH_ARGS=""
|
||||
# We wrap the quickstart call in a timeout, so that we can get logs if the
|
||||
# deploy hangs. 90m = 90 minutes = 1.5 hours
|
||||
/usr/bin/timeout --preserve-status 90m $TRIPLEO_ROOT/tripleo-quickstart/quickstart.sh ${OOOQ_ARGS} 2>&1 \
|
||||
| ts '%Y-%m-%d %H:%M:%S.000 |' | sudo tee /var/log/quickstart_install.log || exit_value=2
|
||||
|
||||
# We use the tripleo.sh pingtest rather than the validate-simple role in
|
||||
# quickstart-extras so that we can easily support the multinode scenario
|
||||
# pingtests. We should add support for that to the validate simple role,
|
||||
# and use that instead.
|
||||
# https://github.com/openstack/tripleo-heat-templates/tree/master/ci/pingtests
|
||||
if [[ -e ${OOO_WORKDIR_LOCAL}/overcloudrc ]]; then
|
||||
$TRIPLEO_CI_DIR/tripleo-ci/scripts/tripleo.sh --overcloud-pingtest 2>&1 | sudo dd of=/var/log/overcloud-pingtest.txt
|
||||
fi
|
||||
|
||||
sudo journalctl -u os-collect-config | sudo tee /var/log/os-collect-config.txt
|
||||
|
||||
# TODO(sshnaidm): to prepare collect-logs role for tripleo-ci specific paths
|
||||
# and remove this function then
|
||||
collect_oooq_logs
|
||||
|
||||
# TODO(sshnaidm): fix this either in role or quickstart.sh
|
||||
# it will not duplicate logs from undercloud and 127.0.0.2
|
||||
sed -i 's/hosts: all:!localhost/hosts: all:!localhost:!127.0.0.2/' $OPT_WORKDIR/playbooks/collect-logs.yml ||:
|
||||
|
||||
# To collect logs from the overcloud nodes we need to use the local ssh config
|
||||
export SSH_CONFIG=$OPT_WORKDIR/ssh.config.local.ansible
|
||||
|
||||
# TODO(sshnaidm): to move postci functionality into collect-logs role
|
||||
$TRIPLEO_ROOT/tripleo-quickstart/quickstart.sh --bootstrap --no-clone \
|
||||
$OOOQ_DEFAULT_ARGS \
|
||||
--requirements requirements.txt \
|
||||
--requirements quickstart-extras-requirements.txt \
|
||||
--config $CONFIG \
|
||||
--playbook collect-logs.yml \
|
||||
-e artcl_collect_dir=/var/log/oooq/collected_logs \
|
||||
-e @$TRIPLEO_ROOT/tripleo-ci/scripts/quickstart/multinode-settings.yml \
|
||||
-e tripleo_root=$TRIPLEO_ROOT \
|
||||
127.0.0.2 2>&1| sudo tee /var/log/quickstart_collectlogs.log ||:
|
||||
|
||||
popd
|
||||
|
||||
echo 'Run completed.'
|
||||
# TODO(sshnaidm): remove this when we're sure there's enough space
|
||||
# Watch free space, the outage could break jobs
|
||||
sudo df -h
|
||||
exit $exit_value
|
||||
Loading…
Reference in New Issue