openstack
/
tripleo-ci
Code Issues Proposed changes
tripleo-ci/scripts/deploy.sh

597 lines
28 KiB
Bash
Executable File
Raw Blame History

set -eux
set -o pipefail
cd
# This sets all the environment variables for undercloud and overcloud installation
source $TRIPLEO_ROOT/tripleo-ci/deploy.env
source $TRIPLEO_ROOT/tripleo-ci/scripts/metrics.bash
source $TRIPLEO_ROOT/tripleo-ci/scripts/common_functions.sh
# Prevent python from buffering stdout, so timestamps are set at appropriate times
export PYTHONUNBUFFERED=true
export DIB_DISTRIBUTION_MIRROR=$NODEPOOL_CENTOS_MIRROR
export STABLE_RELEASE=${STABLE_RELEASE:-""}
# the TLS everywhere job requires the undercloud to have a domain set so it can
# enroll to FreeIPA
if [ $CA_SERVER == 1 ] ; then
# This is needed since we use scripts that are located both in t-h-t and
# tripleo-common for setting up our test CA.
sudo yum install -yq \
openstack-tripleo-heat-templates \
openstack-tripleo-common \
ipa-client \
python-novajoin
export TRIPLEO_DOMAIN=tripleodomain.example.com
export CA_SERVER_HOSTNAME=ipa.$TRIPLEO_DOMAIN
export CA_ADMIN_PASS=$(uuidgen)
export CA_DIR_MANAGER_PASS=$(uuidgen)
export CA_SECRET=$(uuidgen)
export UNDERCLOUD_FQDN=undercloud.$TRIPLEO_DOMAIN
# We can access the CA server through this address for bootstrapping
# purposes.
export CA_SERVER_PRIVATE_IP=$(jq -r '.extra_nodes[0].ips.private[0].addr' ~/instackenv.json)
# Address that will be used for the provisioning interface. The undercloud
# and the overcloud nodes should have access to this.
export CA_SERVER_IP="192.168.24.250"
export CA_SERVER_CIDR="${CA_SERVER_IP}/24"
echo "$CA_SERVER_PRIVATE_IP $CA_SERVER_HOSTNAME" | sudo tee -a /etc/hosts
cat <<EOF >~/freeipa-setup.env
export Hostname=$CA_SERVER_HOSTNAME
export FreeIPAIP=$CA_SERVER_IP
export AdminPassword=$CA_ADMIN_PASS
export DirectoryManagerPassword=$CA_DIR_MANAGER_PASS
export HostsSecret=$CA_SECRET
export UndercloudFQDN=$UNDERCLOUD_FQDN
export ProvisioningCIDR=$CA_SERVER_CIDR
export UsingNovajoin=1
EOF
# Set undercloud FQDN
sudo hostnamectl set-hostname --static $UNDERCLOUD_FQDN
# Copy CA env file and installation script
scp $SSH_OPTIONS ~/freeipa-setup.env centos@$CA_SERVER_PRIVATE_IP:/tmp/freeipa-setup.env
scp $SSH_OPTIONS /usr/share/openstack-tripleo-heat-templates/ci/scripts/freeipa_setup.sh centos@$CA_SERVER_PRIVATE_IP:~/freeipa_setup.sh
# Set up CA
ssh $SSH_OPTIONS -tt centos@$CA_SERVER_PRIVATE_IP "sudo bash ~/freeipa_setup.sh"
# enroll to CA
sudo /usr/libexec/novajoin-ipa-setup \
--principal admin \
--password $CA_ADMIN_PASS \
--server $CA_SERVER_HOSTNAME \
--realm $(echo $TRIPLEO_DOMAIN | awk '{print toupper($0)}') \
--domain $TRIPLEO_DOMAIN \
--hostname $UNDERCLOUD_FQDN \
--otp-file /tmp/ipa-otp.txt \
--precreate
cat <<EOF >$TRIPLEO_ROOT/cloud-names.yaml
parameter_defaults:
CloudDomain: $TRIPLEO_DOMAIN
CloudName: overcloud.$TRIPLEO_DOMAIN
CloudNameInternal: overcloud.internalapi.$TRIPLEO_DOMAIN
CloudNameStorage: overcloud.storage.$TRIPLEO_DOMAIN
CloudNameStorageManagement: overcloud.storagemgmt.$TRIPLEO_DOMAIN
CloudNameCtlplane: overcloud.ctlplane.$TRIPLEO_DOMAIN
EOF
fi
cat <<EOF >$HOME/undercloud-hieradata-override.yaml
ironic::drivers::deploy::http_port: 3816
EOF
echo '[DEFAULT]' > ~/undercloud.conf
echo "hieradata_override = $HOME/undercloud-hieradata-override.yaml" >> ~/undercloud.conf
cat <<EOF >>~/undercloud.conf
network_cidr = 192.168.24.0/24
local_ip = 192.168.24.1/24
network_gateway = 192.168.24.1
undercloud_public_vip = 192.168.24.2
undercloud_admin_vip = 192.168.24.3
masquerade_network = 192.168.24.0/24
dhcp_start = 192.168.24.5
dhcp_end = 192.168.24.30
inspection_iprange = 192.168.24.100,192.168.24.120
EOF
if [ $UNDERCLOUD_SSL == 1 ] ; then
echo 'generate_service_certificate = True' >> ~/undercloud.conf
fi
if [ $UNDERCLOUD_TELEMETRY == 0 ] ; then
echo 'enable_telemetry = False' >> ~/undercloud.conf
echo 'enable_legacy_ceilometer_api = false' >> ~/undercloud.conf
fi
if [ $UNDERCLOUD_UI == 0 ] ; then
echo 'enable_ui = False' >> ~/undercloud.conf
fi
if [ $UNDERCLOUD_VALIDATIONS == 0 ] ; then
echo 'enable_validations = False' >> ~/undercloud.conf
fi
if [ $RUN_TEMPEST_TESTS != 1 ] ; then
echo 'enable_tempest = False' >> ~/undercloud.conf
fi
if [ $CA_SERVER == 1 ] ; then
echo 'enable_novajoin = True' >> ~/undercloud.conf
echo "undercloud_hostname = $UNDERCLOUD_FQDN" >> ~/undercloud.conf
echo "ipa_otp = $(cat /tmp/ipa-otp.txt)" >> ~/undercloud.conf
echo "undercloud_nameservers = $CA_SERVER_IP" >> ~/undercloud.conf
echo "overcloud_domain_name = $TRIPLEO_DOMAIN" >> ~/undercloud.conf
echo "nova::api::vendordata_dynamic_connect_timeout: 20" >> ~/undercloud-hieradata-override.yaml
echo "nova::api::vendordata_dynamic_read_timeout: 20" >> ~/undercloud-hieradata-override.yaml
# NOTE(jaosorior): the DNSServers from the overcloud need to point to the
# CA so the domain can be discovered.
sed -i 's/\(DnsServers: \).*/\1["'$CA_SERVER_IP'", "8.8.8.8"]/' \
$TRIPLEO_ROOT/tripleo-ci/test-environments/network-templates/network-environment.yaml
sed -i 's/\(DnsServers: \).*/\1["'$CA_SERVER_IP'", "8.8.8.8"]/' \
$TRIPLEO_ROOT/tripleo-ci/test-environments/net-iso.yaml
# Use FreeIPA as nameserver
echo -e "nameserver 192.168.24.250\nnameserver 8.8.8.8" | sudo tee /etc/resolv.conf
fi
if [ $UNDERCLOUD_HEAT_CONVERGENCE == 1 ] ; then
cat <<EOF >>$HOME/undercloud-hieradata-override.yaml
heat::engine::convergence_engine: true
EOF
fi
# TODO: fix this in instack-undercloud
sudo mkdir -p /etc/puppet/hieradata
if [ "$OSINFRA" = 1 ]; then
echo "net_config_override = $TRIPLEO_ROOT/tripleo-ci/undercloud-configs/net-config-multinode.json.template" >> ~/undercloud.conf
# Use the dummy network interface if on mitaka
if [ "$STABLE_RELEASE" = "mitaka" ]; then
echo "local_interface = ci-dummy" >> ~/undercloud.conf
fi
fi
# If we're testing an undercloud upgrade, remove the ci repo, since we don't
# want to consume the package being tested until we actually do the upgrade.
if [ "$UNDERCLOUD_MAJOR_UPGRADE" == 1 ] ; then
sudo rm -f /etc/yum.repos.d/delorean-ci.repo
fi
echo "INFO: Check /var/log/undercloud_install.txt for undercloud install output"
echo "INFO: This file can be found in logs/undercloud.tar.xz in the directory containing console.log"
start_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.undercloud.install.seconds"
$TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --undercloud 2>&1 | awk '{ print strftime("%Y-%m-%d %H:%M:%S.000"), "|", $0; fflush(); }' | sudo dd of=/var/log/undercloud_install.txt || (tail -n 50 /var/log/undercloud_install.txt && false)
stop_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.undercloud.install.seconds"
# FreeIPA contains an LDAP server, so we're gonna use that to set up a keystone
# domain that reads from that ldap server.
if [ $CA_SERVER == 1 ] ; then
export LDAP_DOMAIN_NAME=freeipadomain
export LDAP_READER_INIT_PASS=$(uuidgen)
export LDAP_READER_PASS=$(uuidgen)
export DEMO_USER_INIT_PASS=$(uuidgen)
export DEMO_USER_PASS=$(uuidgen)
echo $CA_ADMIN_PASS | kinit admin
echo "$LDAP_READER_INIT_PASS" | ipa user-add keystone --cn="keystone user" \
--first="keystone" --last="user" --password
echo "$DEMO_USER_INIT_PASS" | ipa user-add demo --cn="demo user" \
--first="demo" --last="user" --password
kdestroy -A
# Reset password. Since kerberos prompts for the password to be reset on
# first usage.
echo -e "$LDAP_READER_INIT_PASS\n$LDAP_READER_PASS\n$LDAP_READER_PASS" | kinit keystone
kdestroy -A
echo -e "$DEMO_USER_INIT_PASS\n$DEMO_USER_PASS\n$DEMO_USER_PASS" | kinit demo
kdestroy -A
export LDAP_SUFFIX=$(echo $TRIPLEO_DOMAIN | sed -e 's/^/dc=/' -e 's/\./,dc=/g')
# Create LDAP configuration in heat environment
cat <<EOF >$TRIPLEO_ROOT/keystone-ldap.yaml
parameter_defaults:
KeystoneLDAPDomainEnable: true
KeystoneLDAPBackendConfigs:
$LDAP_DOMAIN_NAME:
url: ldaps://$CA_SERVER_HOSTNAME
user: uid=keystone,cn=users,cn=accounts,$LDAP_SUFFIX
password: $LDAP_READER_PASS
suffix: $LDAP_SUFFIX
user_tree_dn: cn=users,cn=accounts,$LDAP_SUFFIX
user_objectclass: inetOrgPerson
user_id_attribute: uid
user_name_attribute: uid
user_mail_attribute: mail
user_allow_create: false
user_allow_update: false
user_allow_delete: false
group_tree_dn: cn=groups,cn=accounts,$LDAP_SUFFIX
group_objectclass: groupOfNames
group_id_attribute: cn
group_name_attribute: cn
group_member_attribute: member
group_desc_attribute: description
group_allow_create: false
group_allow_update: false
group_allow_delete: false
user_enabled_attribute: nsAccountLock
user_enabled_default: False
user_enabled_invert: true
EOF
fi
if [ "$OVB" = 1 ]; then
# eth1 is on the provisioning netwrok and doesn't have dhcp, so we need to set its MTU manually.
sudo ip link set dev eth1 up
sudo ip link set dev eth1 mtu 1400
echo -e "\ndhcp-option-force=26,1400" | sudo tee -a /etc/dnsmasq-ironic.conf
sudo systemctl restart 'neutron-*'
# The undercloud install is creating file in ~/.cache as root
# change them back so we can build overcloud images
sudo chown -R $USER ~/.cache || true
# check the power status of the last IPMI device we have details for
# this ensures the BMC is ready and sanity tests that its working
PMADDR=$(jq '.nodes[length-1].pm_addr' < ~/instackenv.json | tr '"' ' ')
$TRIPLEO_ROOT/tripleo-ci/scripts/wait_for -d 10 -l 40 -- ipmitool -I lanplus -H $PMADDR -U admin -P password power status
fi
if [ $INTROSPECT == 1 ] ; then
# I'm removing most of the nodes in the env to speed up discovery
# This could be in jq but I don't know how
# Only do this for jobs that use introspection, as it makes the likelihood
# of hitting https://bugs.launchpad.net/tripleo/+bug/1341420 much higher
python -c "import simplejson ; d = simplejson.loads(open(\"instackenv.json\").read()) ; del d[\"nodes\"][$NODECOUNT:] ; print simplejson.dumps(d)" > instackenv_reduced.json
mv instackenv_reduced.json instackenv.json
# Lower the timeout for introspection to decrease failure time
# It should not take more than 10 minutes with IPA ramdisk and no extra collectors
sudo sed -i '2itimeout = 600' /etc/ironic-inspector/inspector.conf
sudo systemctl restart openstack-ironic-inspector
fi
if [ $NETISO_V4 -eq 1 ] || [ $NETISO_V6 -eq 1 ]; then
# Update our floating range to use a 10. /24
export FLOATING_IP_CIDR=${FLOATING_IP_CIDR:-"10.0.0.0/24"}
export FLOATING_IP_START=${FLOATING_IP_START:-"10.0.0.100"}
export FLOATING_IP_END=${FLOATING_IP_END:-"10.0.0.200"}
export EXTERNAL_NETWORK_GATEWAY=${EXTERNAL_NETWORK_GATEWAY:-"10.0.0.1"}
# Make our undercloud act as the external gateway
# OVB uses eth2 as the "external" network
# NOTE: seed uses eth0 for the local network.
cat >> /tmp/eth2.cfg <<EOF_CAT
network_config:
- type: interface
name: eth2
use_dhcp: false
addresses:
- ip_netmask: 10.0.0.1/24
- ip_netmask: 2001:db8:fd00:1000::1/64
EOF_CAT
sudo os-net-config -c /tmp/eth2.cfg -v
fi
if [ "$OSINFRA" = "0" ]; then
# Our ci underclouds don't have enough RAM to allow us to use a tmpfs
export DIB_NO_TMPFS=1
# No point waiting for a grub prompt in ci
export DIB_GRUB_TIMEOUT=0
# Override the default repositories set by tripleo.sh, to add the delorean-ci repository
export OVERCLOUD_IMAGES_DIB_YUM_REPO_CONF=$(ls /etc/yum.repos.d/delorean*)
# Directing the output of this command to a file as its extreemly verbose
echo "INFO: Check /var/log/image_build.txt for image build output"
echo "INFO: This file can be found in logs/undercloud.tar.xz in the directory containing console.log"
start_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.overcloud.images.seconds"
$TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --overcloud-images 2>&1 | awk '{ print strftime("%Y-%m-%d %H:%M:%S.000"), "|", $0; fflush(); }' | sudo dd of=/var/log/image_build.txt || (tail -n 50 /var/log/image_build.txt && false)
stop_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.overcloud.images.seconds"
OVERCLOUD_IMAGE_MB=$(du -ms overcloud-full.qcow2 | cut -f 1)
record_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.overcloud.image.size_mb" "$OVERCLOUD_IMAGE_MB"
start_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.register.nodes.seconds"
if [ $INTROSPECT == 1 ]; then
export INTROSPECT_NODES=1
fi
$TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --register-nodes
# We don't want to keep this set for further calls to tripleo.sh
unset INTROSPECT_NODES
stop_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.register.nodes.seconds"
if [ $INTROSPECT == 1 ] ; then
start_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.introspect.seconds"
$TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --introspect-nodes
stop_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.introspect.seconds"
fi
if [ $PREDICTABLE_PLACEMENT == 1 ]; then
source ~/stackrc
NODE_ID_0=$(ironic node-list | grep available | head -n 1 | tail -n 1 | awk '{print $2}')
NODE_ID_1=$(ironic node-list | grep available | head -n 2 | tail -n 1 | awk '{print $2}')
NODE_ID_2=$(ironic node-list | grep available | head -n 3 | tail -n 1 | awk '{print $2}')
NODE_ID_3=$(ironic node-list | grep available | head -n 4 | tail -n 1 | awk '{print $2}')
ironic node-update $NODE_ID_0 replace properties/capabilities='node:controller-0,boot_option:local'
ironic node-update $NODE_ID_1 replace properties/capabilities='node:controller-1,boot_option:local'
ironic node-update $NODE_ID_2 replace properties/capabilities='node:controller-2,boot_option:local'
ironic node-update $NODE_ID_3 replace properties/capabilities='node:compute-0,boot_option:local'
fi
sleep 60
fi
if [ -n "${OVERCLOUD_UPDATE_ARGS:-}" ] ; then
# Reinstall openstack-tripleo-heat-templates from delorean-current.
# Since we're testing updates, we want to remove any version we may have
# installed from the delorean-ci repo and install from delorean-current,
# or just delorean in the case of stable branches.
sudo rpm -ev --nodeps openstack-tripleo-heat-templates
sudo yum -y --disablerepo=* --enablerepo=delorean,delorean-current install openstack-tripleo-heat-templates
fi
if [ "$MULTINODE" = "1" ]; then
# Start the script that will configure os-collect-config on the subnodes
source ~/stackrc
if [ "$OVERCLOUD_MAJOR_UPGRADE" == 1 ] ; then
# Download the previous release openstack-tripleo-heat-templates to a directory
# we then deploy this and later upgrade to the default --templates location
# FIXME - we should make the tht-compat package work here instead
OLD_THT=$(curl https://trunk.rdoproject.org/centos7-$UPGRADE_RELEASE/current/ | grep "openstack-tripleo-heat-templates" | grep "noarch.rpm" | grep -v "tripleo-heat-templates-compat" | sed "s/^.*>openstack-tripleo-heat-templates/openstack-tripleo-heat-templates/" | cut -d "<" -f1)
echo "Downloading https://trunk.rdoproject.org/centos7-$UPGRADE_RELEASE/current/$OLD_THT"
rm -fr $TRIPLEO_ROOT/$UPGRADE_RELEASE/*
mkdir -p $TRIPLEO_ROOT/$UPGRADE_RELEASE
curl -o $TRIPLEO_ROOT/$UPGRADE_RELEASE/$OLD_THT https://trunk.rdoproject.org/centos7-$UPGRADE_RELEASE/current/$OLD_THT
pushd $TRIPLEO_ROOT/$UPGRADE_RELEASE
rpm2cpio openstack-tripleo-heat-templates-*.rpm | cpio -ivd
popd
# Backup current deploy args:
UPGRADE_OVERCLOUD_DEPLOY_ARGS=$OVERCLOUD_DEPLOY_ARGS
# Rewrite all template paths to ensure paths match the
# --templates location
TEMPLATE_PATH="/usr/share/openstack-tripleo-heat-templates"
ENV_PATH="-e $TEMPLATE_PATH"
STABLE_TEMPLATE_PATH="$TRIPLEO_ROOT/$UPGRADE_RELEASE/usr/share/openstack-tripleo-heat-templates"
STABLE_ENV_PATH="-e $STABLE_TEMPLATE_PATH"
echo "SHDEBUG OVERCLOUD_DEPLOY_ARGS BEFORE=$OVERCLOUD_DEPLOY_ARGS"
UPGRADE_OVERCLOUD_DEPLOY_ARGS=${UPGRADE_OVERCLOUD_DEPLOY_ARGS//$STABLE_ENV_PATH/$ENV_PATH}
OVERCLOUD_DEPLOY_ARGS=${OVERCLOUD_DEPLOY_ARGS//$ENV_PATH/$STABLE_ENV_PATH}
echo "UPGRADE_OVERCLOUD_DEPLOY_ARGS=$UPGRADE_OVERCLOUD_DEPLOY_ARGS"
echo "OVERCLOUD_DEPLOY_ARGS=$OVERCLOUD_DEPLOY_ARGS"
# Set deploy args for stable deployment:
export OVERCLOUD_DEPLOY_ARGS="$OVERCLOUD_DEPLOY_ARGS --templates $STABLE_TEMPLATE_PATH -e $STABLE_TEMPLATE_PATH/environments/deployed-server-environment.yaml -e $STABLE_TEMPLATE_PATH/environments/services/sahara.yaml"
if [ ! -z $UPGRADE_ENV ]; then
export OVERCLOUD_DEPLOY_ARGS="$OVERCLOUD_DEPLOY_ARGS -e $TRIPLEO_ROOT/$UPGRADE_RELEASE/$UPGRADE_ENV"
fi
echo_vars_to_deploy_env
$TRIPLEO_ROOT/$UPGRADE_RELEASE/usr/share/openstack-tripleo-heat-templates/deployed-server/scripts/get-occ-config.sh 2>&1 | sudo dd of=/var/log/deployed-server-os-collect-config.log &
else
/usr/share/openstack-tripleo-heat-templates/deployed-server/scripts/get-occ-config.sh 2>&1 | sudo dd of=/var/log/deployed-server-os-collect-config.log &
fi
# Create dummy overcloud-full image since there is no way (yet) to disable
# this constraint in the heat templates
if ! openstack image show overcloud-full; then
qemu-img create -f qcow2 overcloud-full.qcow2 1G
glance image-create \
--container-format bare \
--disk-format qcow2 \
--name overcloud-full \
--file overcloud-full.qcow2
fi
fi
if [ $OVERCLOUD == 1 ] ; then
source ~/stackrc
start_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.overcloud.deploy.seconds"
http_proxy= $TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --overcloud-deploy ${TRIPLEO_SH_ARGS:-}
stop_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.overcloud.deploy.seconds"
# Add hosts to /etc/hosts
openstack stack output show overcloud HostsEntry -f value -c output_value | sudo tee -a /etc/hosts
fi
if [ $UNDERCLOUD_IDEMPOTENT == 1 ]; then
echo "INFO: Check /var/log/undercloud_install_idempotent.txt for undercloud install output"
echo "INFO: This file can be found in logs/undercloud.tar.xz in the directory containing console.log"
$TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --undercloud 2>&1 | sudo dd of=/var/log/undercloud_install_idempotent.txt || (tail -n 50 /var/log/undercloud_install_idempotent.txt && false)
fi
if [ -n "${OVERCLOUD_UPDATE_ARGS:-}" ] ; then
# Reinstall openstack-tripleo-heat-templates, this will pick up the version
# from the delorean-ci repo if the patch being tested is from
# tripleo-heat-templates, otherwise it will just reinstall from
# delorean-current.
sudo rpm -ev --nodeps openstack-tripleo-heat-templates
sudo yum -y install openstack-tripleo-heat-templates
start_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.overcloud.update.seconds"
http_proxy= $TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --overcloud-update ${TRIPLEO_SH_ARGS:-}
stop_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.overcloud.update.seconds"
fi
if [ "$MULTINODE" == 0 ] && [ "$OVERCLOUD" == 1 ] ; then
# Sanity test we deployed what we said we would
source ~/stackrc
[ "$NODECOUNT" != $(nova list | grep ACTIVE | wc -l | cut -f1 -d " ") ] && echo "Wrong number of nodes deployed" && exit 1
if [ $PREDICTABLE_PLACEMENT == 1 ]; then
# Verify our public VIP is the one we specified
grep -q 10.0.0.9 ~/overcloudrc || (echo "Wrong public vip deployed " && exit 1)
# Verify our specified hostnames were used
INSTANCE_ID_0=$(nova list | grep controller-0-tripleo-ci-a-foo | awk '{print $2}')
INSTANCE_ID_1=$(nova list | grep controller-1-tripleo-ci-b-bar | awk '{print $2}')
INSTANCE_ID_2=$(nova list | grep controller-2-tripleo-ci-c-baz | awk '{print $2}')
INSTANCE_ID_3=$(nova list | grep compute-0-tripleo-ci-a-test | awk '{print $2}')
# Verify the correct ironic nodes were used
echo "Verifying predictable placement configuration was honored."
ironic node-list | grep $INSTANCE_ID_0 | grep -q $NODE_ID_0 || (echo "$INSTANCE_ID_0 not deployed to node $NODE_ID_0" && exit 1)
ironic node-list | grep $INSTANCE_ID_1 | grep -q $NODE_ID_1 || (echo "$INSTANCE_ID_1 not deployed to node $NODE_ID_1" && exit 1)
ironic node-list | grep $INSTANCE_ID_2 | grep -q $NODE_ID_2 || (echo "$INSTANCE_ID_2 not deployed to node $NODE_ID_2" && exit 1)
ironic node-list | grep $INSTANCE_ID_3 | grep -q $NODE_ID_3 || (echo "$INSTANCE_ID_3 not deployed to node $NODE_ID_3" && exit 1)
echo "Verified."
fi
if [ $PACEMAKER == 1 ] ; then
# Wait for the pacemaker cluster to settle and all resources to be
# available. heat-{api,engine} are the best candidates since due to the
# constraint ordering they are typically started last. We'll wait up to
# 180s.
start_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.overcloud.settle.seconds"
timeout -k 10 240 ssh $SSH_OPTIONS heat-admin@$(nova list | grep controller-0 | awk '{print $12}' | cut -d'=' -f2) sudo crm_resource -r openstack-heat-api --wait || {
exitcode=$?
echo "crm_resource for openstack-heat-api has failed!"
exit $exitcode
}
timeout -k 10 240 ssh $SSH_OPTIONS heat-admin@$(nova list | grep controller-0 | awk '{print $12}' | cut -d'=' -f2) sudo crm_resource -r openstack-heat-engine --wait|| {
exitcode=$?
echo "crm_resource for openstack-heat-engine has failed!"
exit $exitcode
}
stop_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.overcloud.settle.seconds"
fi
fi
if [ -f ~/overcloudrc ]; then
source ~/overcloudrc
fi
if [ "$OVERCLOUD_MAJOR_UPGRADE" == 1 ] ; then
# Re-enable the delorean-ci repo, as ZUUL_REFS,
# and thus the contents of delorean-ci may contain packages
# we want to test for the current branch on upgrade
if [ -s /etc/nodepool/sub_nodes_private ]; then
for ip in $(cat /etc/nodepool/sub_nodes_private); do
ssh $SSH_OPTIONS -tt -i /etc/nodepool/id_rsa $ip \
sudo sed -i -e \"s/enabled=0/enabled=1/\" /etc/yum.repos.d/delorean-ci.repo
done
fi
source ~/stackrc
# Set deploy args for stable deployment:
# We have to use the backward compatible
if [ ! -z $UPGRADE_ENV ]; then
export OVERCLOUD_DEPLOY_ARGS="$OVERCLOUD_DEPLOY_ARGS -e $UPGRADE_ENV"
fi
# update-from-deployed-server-$UPGRADE_RELEASE.yaml environment when upgrading from
# $UPGRADE_RELEASE.
export OVERCLOUD_DEPLOY_ARGS="$UPGRADE_OVERCLOUD_DEPLOY_ARGS -e /usr/share/openstack-tripleo-heat-templates/environments/deployed-server-environment.yaml -e /usr/share/openstack-tripleo-heat-templates/environments/services/sahara.yaml"
if [ "$UPGRADE_RELEASE" == "newton" ]; then
export OVERCLOUD_DEPLOY_ARGS="$OVERCLOUD_DEPLOY_ARGS -e /usr/share/openstack-tripleo-heat-templates/environments/updates/update-from-deployed-server-$UPGRADE_RELEASE.yaml"
fi
if [ ! -z $UPGRADE_ENV ]; then
export OVERCLOUD_DEPLOY_ARGS="$OVERCLOUD_DEPLOY_ARGS -e $UPGRADE_ENV"
fi
echo_vars_to_deploy_env
if [ "$MULTINODE" = "1" ]; then
/usr/share/openstack-tripleo-heat-templates/deployed-server/scripts/get-occ-config.sh 2>&1 | sudo dd of=/var/log/deployed-server-os-collect-config-22.log &
fi
# We run basic sanity tests before/after, which includes creating some resources which
# must survive the upgrade. The upgrade is performed in two steps, even though this
# is an all-in-one test, as this is close to how a real deployment with computes would
# be upgraded.
$TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --overcloud-sanity --skip-sanitytest-cleanup
$TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --overcloud-upgrade
$TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --overcloud-sanity --skip-sanitytest-create --skip-sanitytest-cleanup
$TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --overcloud-upgrade-converge
$TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --overcloud-sanity --skip-sanitytest-create
fi
if [ $CA_SERVER == 1 ] ; then
source ~/overcloudrc
# Verify that the domain exists
openstack domain show $LDAP_DOMAIN_NAME
# Add admin role to admin user for freeipadomain
openstack role add admin --domain $LDAP_DOMAIN_NAME --user admin --user-domain Default
# Verify we can access the users in the given domain
openstack user list --domain $LDAP_DOMAIN_NAME
# Add demo project to domain
openstack project create demo --domain $LDAP_DOMAIN_NAME
# Add admin role to user for that project
openstack role add admin --project demo --project-domain $LDAP_DOMAIN_NAME \
--user demo --user-domain $LDAP_DOMAIN_NAME
# Create rc file for demo user
cat <<EOF >~/overcloudrc.demouser
# Clear any old environment that may conflict.
for key in \$( set | awk '{FS="="} /^OS_/ {print \$1}' ); do unset \$key ; done
export OS_USERNAME=demo
export OS_USER_DOMAIN_NAME=$LDAP_DOMAIN_NAME
export OS_PROJECT_DOMAIN_NAME=$LDAP_DOMAIN_NAME
export OS_BAREMETAL_API_VERSION=1.29
export NOVA_VERSION=1.1
export OS_PROJECT_NAME=demo
export OS_PASSWORD=$DEMO_USER_PASS
export OS_NO_CACHE=True
export COMPUTE_API_VERSION=1.1
export no_proxy=,overcloud.$TRIPLEO_DOMAIN,overcloud.ctlplane.$TRIPLEO_DOMAIN,overcloud.$TRIPLEO_DOMAIN,overcloud.ctlplane.$TRIPLEO_DOMAIN
export OS_CLOUDNAME=overcloud
export OS_AUTH_URL=https://overcloud.$TRIPLEO_DOMAIN:13000/v3
export IRONIC_API_VERSION=1.29
export OS_IDENTITY_API_VERSION=3
export OS_AUTH_TYPE=password
export PYTHONWARNINGS="ignore:Certificate has no, ignore:A true SSLContext object is not available"
EOF
# Tell tripleo.sh/pingtest to use demouserrc instead of overcloudrc. Note
# that we don't include the $HOME path prefix on the variable, as this is
# implicitly added in tripleo.sh
export ALT_OVERCLOUDRC=overcloudrc.demouser
fi
if [ $RUN_PING_TEST == 1 ] ; then
start_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.overcloud.ping_test.seconds"
OVERCLOUD_PINGTEST_OLD_HEATCLIENT=0 $TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --overcloud-pingtest $OVERCLOUD_PINGTEST_ARGS
stop_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.overcloud.ping_test.seconds"
fi
if [ $RUN_TEMPEST_TESTS == 1 ] ; then
start_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.overcloud.tempest.seconds"
export TEMPEST_REGEX='^(?=(.*smoke))(?!('
export TEMPEST_REGEX="${TEMPEST_REGEX}tempest.scenario.test_volume_boot_pattern" # http://bugzilla.redhat.com/1272289
export TEMPEST_REGEX="${TEMPEST_REGEX}|tempest.api.identity.*v3" # https://bugzilla.redhat.com/1266947
export TEMPEST_REGEX="${TEMPEST_REGEX}|.*test_external_network_visibility" # https://bugs.launchpad.net/tripleo/+bug/1577769
export TEMPEST_REGEX="${TEMPEST_REGEX}|tempest.api.data_processing" # Sahara is not enabled by default and has problem with performance
export TEMPEST_REGEX="${TEMPEST_REGEX}))"
bash $TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --run-tempest
stop_metric "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.overcloud.tempest.seconds"
fi
if [ $TEST_OVERCLOUD_DELETE -eq 1 ] ; then
source ~/stackrc
$TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --overcloud-delete
fi
# Upgrade part
if [ "$UNDERCLOUD_MAJOR_UPGRADE" == 1 ] ; then
# Reset or unset STABLE_RELEASE so that we upgrade to the next major
# version
if [ "$STABLE_RELEASE" = "queens" ]; then
# TODO: switch STABLE_RELEASE to rocky when released
export STABLE_RELEASE=""
elif [ "$STABLE_RELEASE" = "pike" ]; then
export STABLE_RELEASE="queens"
elif [ "$STABLE_RELEASE" = "ocata" ]; then
export STABLE_RELEASE="pike"
elif [ "$STABLE_RELEASE" = "mitaka" ]; then
export STABLE_RELEASE="newton"
elif [ "$STABLE_RELEASE" = "newton" ]; then
export STABLE_RELEASE="ocata"
fi
echo_vars_to_deploy_env
# Add the delorean ci repo so that we include the package being tested
layer_ci_repo
$TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --undercloud-upgrade 2>&1 | awk '{ print strftime("%Y-%m-%d %H:%M:%S.000"), "|", $0; fflush(); }' | sudo dd of=/var/log/undercloud_upgrade.txt || (tail -n 50 /var/log/undercloud_upgrade.txt && false)
fi
View Git Blame Copy Permalink