From 9e391e8a0fda02fc9762f09a539834d9abede493 Mon Sep 17 00:00:00 2001 From: Ben Kero Date: Tue, 3 Jan 2017 11:19:19 +0900 Subject: [PATCH] Add OOOQ multinode test support This commit adds support to run nodepool multinode tests using tripleo-quickstart as the runner. Change-Id: I11827893c716faf2d295f86ac747470e45613505 Depends-On: I0688d7538d4c6fe4d24d5152ee4c8dae2557508f --- scripts/quickstart/multinode-playbook.yml | 34 +++++ scripts/quickstart/multinode-settings.yml | 141 ++++++++++++++++++ toci_gate_test.sh | 14 +- toci_instack_oooq_multinode.sh | 172 ++++++++++++++++++++++ 4 files changed, 352 insertions(+), 9 deletions(-) create mode 100644 scripts/quickstart/multinode-playbook.yml create mode 100644 scripts/quickstart/multinode-settings.yml create mode 100755 toci_instack_oooq_multinode.sh diff --git a/scripts/quickstart/multinode-playbook.yml b/scripts/quickstart/multinode-playbook.yml new file mode 100644 index 000000000..8bb1784f0 --- /dev/null +++ b/scripts/quickstart/multinode-playbook.yml @@ -0,0 +1,34 @@ +--- +- include: quickstart.yml + +- name: Prepare the undercloud for installation + hosts: undercloud + roles: + - undercloud-setup + tags: + - undercloud-setup + environment: + WORKSPACE: "{{ lookup('env','WORKSPACE') }}" + SSH_OPTIONS: "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=Verbose -o PasswordAuthentication=no -o ConnectionAttempts=32 -l jenkins" + +- name: Install the undercloud + hosts: undercloud + roles: + - undercloud-deploy + +- name: Validate the undercloud + hosts: undercloud + roles: + - validate-undercloud + +- name: Run overcloud prep roles + hosts: undercloud + gather_facts: no + roles: + - { role: overcloud-prep-images, step_overcloud_image: false, step_glance_upload: true, step_register: false } + +- name: Deploy the overcloud + hosts: undercloud + gather_facts: no + roles: + - overcloud-deploy diff --git a/scripts/quickstart/multinode-settings.yml b/scripts/quickstart/multinode-settings.yml new file mode 100644 index 000000000..0199e6c1b --- /dev/null +++ b/scripts/quickstart/multinode-settings.yml @@ -0,0 +1,141 @@ +extra_args: '-e /usr/share/openstack-tripleo-heat-templates/environments/deployed-server-environment.yaml \ + -e ${TRIPLEO_ROOT}/tripleo-ci/test-environments/multinode.yaml \ + -e ${TRIPLEO_ROOT}/tripleo-ci/test-environments/worker-config.yaml \ + --compute-scale 0' +ssl_overcloud: false +network_isolation: false +flavor_args: "" +undercloud_local_interface: "br-ex" +undercloud_generate_service_certificate: false +undercloud_check_idempotency: true +undercloud_check_sanity: true + +toci_vxlan_networking: true +toci_vxlan_networking_multinode: true +hostname_correction: true +package_installs: true +non_root_user_setup: true + +step_overcloud_image: false +step_glance_upload: true +step_register: false + +hypervisor_wait: false +deployed_server: true +validation_args: "--validation-errors-nonfatal" +deploy_timeout: 30 + +undercloud_conf_extras: "net_config_override = /opt/stack/new/tripleo-ci/undercloud-configs/net-config-multinode.json.template" + +enable_vbmc: false + +# TODO(trown): Create repo configs for each release in the release configs +repos: + - type: generic + reponame: delorean + filename: delorean.repo + baseurl: http://buildlogs.centos.org/centos/7/cloud/x86_64/rdo-trunk-master-tripleo/ + hash_url: http://buildlogs.centos.org/centos/7/cloud/x86_64/rdo-trunk-master-tripleo/delorean.repo + priority: 20 + + - type: generic + reponame: delorean-current + filename: delorean-current.repo + baseurl: http://trunk.rdoproject.org/centos7/current + hash_url: http://trunk.rdoproject.org/centos7/current/delorean.repo + priority: 10 + includepkgs: + - diskimage-builder + - instack + - instack-undercloud + - os-apply-config + - os-cloud-config + - os-collect-config + - os-net-config + - os-refresh-config + - python-tripleoclient + - openstack-tripleo-common + - openstack-tripleo-heat-templates + - openstack-tripleo-image-elements + - openstack-tripleo + - openstack-tripleo-puppet-elements + - openstack-puppet-modules + - openstack-tripleo-ui + - puppet-* + + - type: file + filename: delorean-deps.repo + down_url: http://trunk.rdoproject.org/centos7-master/delorean-deps.repo + + - type: package + pkg_name: centos-release-ceph-jewel + custom_cmd: 'yum install -y --enablerepo=extras' + +artcl_publish: false +artcl_rsync_logs: false +artcl_collect_list: + - /var/log/ + - /var/lib/mysql + - /home/jenkins/.instack/install-undercloud.log + - /home/jenkins/stackrc + - /home/jenkins/overcloudrc + - /home/jenkins/*.log + - /home/jenkins/*.txt + - /home/jenkins/*.json + - /home/jenkins/*.conf + - /home/jenkins/*.yml + - /home/jenkins/*.yaml + - /home/jenkins/*.sh + - /home/jenkins/deploy-overcloudrc + - /home/jenkins/network-environment.yaml + - /home/jenkins/tempest/*.xml + - /home/jenkins/tempest/*.html + - /home/jenkins/tempest/*.log + - /home/jenkins/tempest/*.sh + - /home/jenkins/tempest/etc/*.conf + - /home/jenkins/local_tht + - /usr/share/openstack-tripleo-heat-templates + - /tmp/*.yml + - /tmp/*.yaml + - /etc/aodh + - /etc/ceilometer + - /etc/cinder + - /etc/dnsmasq-ironic.conf + - /etc/docker-registry.yml + - /etc/glance + - /etc/gnocchi + - /etc/heat + - /etc/haproxy + - /etc/httpd + - /etc/ironic + - /etc/ironic-inspector + - /etc/keepalived + - /etc/keystone + - /etc/libvirt + - /etc/manila + - /etc/mistral + - /etc/mongod.conf + - /etc/mongos.conf + - /etc/neutron + - /etc/nova + - /etc/ntp + - /etc/ntp.conf + - /etc/os-collect-config.conf + - /etc/os-net-config + - /etc/puppet + - /etc/qpid + - /etc/qpidd.conf + - /etc/rabbitmq + - /etc/redis + - /etc/redis.conf* + - /etc/resolv.conf + - /etc/sahara + - /etc/selinux + - /etc/swift + - /etc/sysconfig + - /etc/systemd + - /etc/trove + - /etc/yum.repos.d + - /etc/zaqar +artcl_gzip_only: true + diff --git a/toci_gate_test.sh b/toci_gate_test.sh index 8b67cb835..1499bb5c5 100755 --- a/toci_gate_test.sh +++ b/toci_gate_test.sh @@ -105,14 +105,6 @@ elif [[ "$TOCI_JOBTYPE" =~ "periodic" && "$TOCI_JOBTYPE" =~ "-nonha" ]]; then UNDERCLOUD_IDEMPOTENT=1 fi -# TODO(trown) remove this when we get the multinode-oooq job[1] in a reasonable -# state. Right now it runs on every quickstart and quickstart-extras patch -# wasting resources. -# [1] https://review.openstack.org/#/c/416110 -if [[ $TOCI_JOBTYPE =~ multinode-oooq ]]; then - exit 1 -fi - # start dstat early # TODO add it to the gate image building sudo yum install -y dstat nmap-ncat #nc is for metrics @@ -252,7 +244,11 @@ for JOB_TYPE_PART in $(sed 's/-/ /g' <<< "${TOCI_JOBTYPE:-}") ; do ;; oooq) export OOOQ=1 - TOCIRUNNER="./toci_instack_oooq.sh" + if [[ "$TOCI_JOBTYPE" =~ "multinode" ]]; then + TOCIRUNNER="./toci_instack_oooq_multinode.sh" + else + TOCIRUNNER="./toci_instack_oooq.sh" + fi ;; esac done diff --git a/toci_instack_oooq_multinode.sh b/toci_instack_oooq_multinode.sh new file mode 100755 index 000000000..c36104b41 --- /dev/null +++ b/toci_instack_oooq_multinode.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +set -eux +set -o pipefail + +## Signal to toci_gate_test.sh we've started by +touch /tmp/toci.started + +exit_value=0 +export CURRENT_DIR=$(dirname ${BASH_SOURCE[0]:-$0}) +export TRIPLEO_CI_DIR=$CURRENT_DIR/../ + +# TODO(sshnaidm): remove this immediately when settings are in yaml files +source $TRIPLEO_CI_DIR/tripleo-ci/scripts/common_vars.bash +source $TRIPLEO_CI_DIR/tripleo-ci/scripts/common_functions.sh +#source $TRIPLEO_CI_DIR/tripleo-ci/scripts/metrics.bash + +# Clear out any puppet modules on the node placed their by infra configuration +sudo rm -rf /etc/puppet/modules/* + +# Copy nodepool public key to jenkins user +sudo cp /etc/nodepool/id_rsa.pub $HOME/.ssh/ +sudo chown $USER:$USER $HOME/.ssh/id_rsa.pub + +export USE_DELOREAN=0 +# TODO(sshnaidm): Hack for gate role, the excluded repos should be handled +# properly there. To fix ansible gate role. +export ZUUL_CHANGES=${ZUUL_CHANGES:-} +export ZUUL_CHANGES=$(echo $ZUUL_CHANGES | python -c 'import sys; print "^".join([i for i in sys.stdin.readline().split("^") if "openstack-infra/tripleo-ci:" not in i])') +if [[ -n "$ZUUL_CHANGES" ]]; then + export ZUUL_HOST="review.openstack.org" + export USE_DELOREAN=1 +fi + +# TODO(sshnaidm): To create tripleo-ci special yaml config files in oooq +# for every TOCI_JOBTYPE, i.e. ovb-nonha-ipv6.yml +if [[ "$TOCI_JOBTYPE" =~ "-ha" ]]; then + CONFIG=${CONFIG:-"$TRIPLEO_ROOT/tripleo-quickstart/config/general_config/ha.yml"} +elif [[ "$TOCI_JOBTYPE" =~ "-nonha" ]]; then + CONFIG=${CONFIG:-"$TRIPLEO_ROOT/tripleo-quickstart/config/general_config/minimal.yml"} +else + CONFIG=${CONFIG:-"$TRIPLEO_ROOT/tripleo-quickstart/config/general_config/minimal.yml"} +fi + +# Add jenkin user's SSH key to root authorized_keys for Ansible to run +sudo mkdir -p /root/.ssh/ +sudo cp ${HOME}/.ssh/authorized_keys /root/.ssh/ +sudo chmod 0600 /root/.ssh/authorized_keys +sudo chown root:root /root/.ssh/authorized_keys + +# TODO(bkero): Use an ansible role to create this file +sudo yum install -y qemu-img +qemu-img create -f qcow2 /home/jenkins/overcloud-full.qcow2 1G + +# Bootstrap the subnodes +$TRIPLEO_CI_DIR/tripleo-ci/scripts/tripleo.sh --bootstrap-subnodes 2>&1 | sudo dd of=/var/log/bootstrap-subnodes.log || (tail -n 50 /var/log/bootstrap-subnodes.log && false) + +# Install our test cert so SSL tests work +sudo cp $TRIPLEO_ROOT/tripleo-ci/test-environments/overcloud-cacert.pem /etc/pki/ca-trust/source/anchors/ +sudo cp $TRIPLEO_ROOT/tripleo-ci/test-environments/overcloud-cacert-ipv6.pem /etc/pki/ca-trust/source/anchors/ +sudo update-ca-trust extract + +# TODO(sshnaidm): to move these variables to jobs yaml config files (see above) +if [[ "${STABLE_RELEASE}" =~ ^(liberty|mitaka)$ ]] ; then + export OVERCLOUD_DEPLOY_ARGS="$OVERCLOUD_DEPLOY_ARGS -e $TRIPLEO_ROOT/tripleo-ci/test-environments/worker-config-mitaka-and-below.yaml" +else + export OVERCLOUD_DEPLOY_ARGS="$OVERCLOUD_DEPLOY_ARGS -e $TRIPLEO_ROOT/tripleo-ci/test-environments/worker-config.yaml -e /usr/share/openstack-tripleo-heat-templates/environments/low-memory-usage.yaml" +fi +export OPT_WORKDIR=${WORKSPACE}/.quickstart +export OOOQ_LOGS=/var/log/oooq +export OOO_WORKDIR_LOCAL=$HOME +export OOOQ_DEFAULT_ARGS=" --working-dir $OPT_WORKDIR --retain-inventory -T none -e working_dir=$OOO_WORKDIR_LOCAL -R ${STABLE_RELEASE:-master}" + +export OOOQ_ARGS=" --working-dir ${OPT_WORKDIR} \ + -e working_dir=${OOO_WORKDIR_LOCAL} \ + --bootstrap \ + --no-clone \ + --retain-inventory \ + --tags undercloud-setup,undercloud-scripts,undercloud-install,undercloud-post-install,overcloud-scripts,overcloud-deploy \ + --teardown none \ + --release ${STABLE_RELEASE:-master} \ + --config config/general_config/minimal.yml \ + --extra-vars @${TRIPLEO_ROOT}/tripleo-ci/scripts/quickstart/multinode-settings.yml \ + --playbook multinode-playbook.yml \ + --requirements requirements.txt \ + --requirements quickstart-extras-requirements.txt \ + 127.0.0.2" + +#shopt -s extglob +#rm -rf /opt/stack/new/!(tripleo-ci|tripleo-quickstart|tripleo-quickstart-extras) +# End of cleaning +# HINT: If there's no enough space, remove swap file in /root/ + +# TODO(sshnaidm): when collect-logs role will have the same functionality, +# replace postci function with this role (see in the end of file). +trap "exit_val=\$?; [ \$exit_val != 0 ] && echo ERROR DURING PREVIOUS COMMAND ^^^ && echo 'See postci.txt in the logs directory for debugging details'; postci \$exit_val 2>&1 | ts '%Y-%m-%d %H:%M:%S.000 |' > $WORKSPACE/logs/postci.log 2>&1" EXIT + +mkdir -p $WORKSPACE/logs + +export IP_DEVICE=${IP_DEVICE:-"eth0"} +MY_IP=$(ip addr show dev $IP_DEVICE | awk '/inet / {gsub("/.*", "") ; print $2}') +MY_IP_eth1=$(ip addr show dev eth1 | awk '/inet / {gsub("/.*", "") ; print $2}') || MY_IP_eth1="" + +export http_proxy="" +undercloud_net_range="192.168.24." +undercloud_services_ip=$undercloud_net_range"1" +undercloud_haproxy_public_ip=$undercloud_net_range"2" +undercloud_haproxy_admin_ip=$undercloud_net_range"3" +export no_proxy=$undercloud_services_ip,$undercloud_haproxy_public_ip,$undercloud_haproxy_admin_ip,$MY_IP,$MY_IP_eth1 + +[[ ! -e $OPT_WORKDIR ]] && mkdir -p $OPT_WORKDIR && sudo chown -R ${USER} $OPT_WORKDIR +sudo mkdir $OOOQ_LOGS && sudo chown -R ${USER} $OOOQ_LOGS + +# make the requirements point to local checkout of tripleo-quickstart-extras +echo "file://${TRIPLEO_ROOT}/tripleo-quickstart-extras/#egg=tripleo-quickstart-extras" > ${TRIPLEO_ROOT}/tripleo-quickstart/quickstart-extras-requirements.txt + +# TODO(bkero): Correct this with a dynamic inventory configuration +sed -i 's/^undercloud ansible_host=undercloud/undercloud ansible_host=127.0.0.2/' $TRIPLEO_ROOT/tripleo-ci/scripts/hosts + +cp $TRIPLEO_ROOT/tripleo-ci/scripts/hosts $OPT_WORKDIR/hosts +cp $TRIPLEO_ROOT/tripleo-ci/scripts/quickstart/*yml $TRIPLEO_ROOT/tripleo-quickstart/playbooks/ +$TRIPLEO_ROOT/tripleo-quickstart/quickstart.sh --install-deps + +pushd $TRIPLEO_ROOT/tripleo-quickstart/ +# TODO(sshnaidm): fix inventory role with prepares ssh.config.ansible, +# it's not usable here right now. Hopefully ssh config is not required for us. +export ANSIBLE_SSH_ARGS="" +# We wrap the quickstart call in a timeout, so that we can get logs if the +# deploy hangs. 90m = 90 minutes = 1.5 hours +/usr/bin/timeout --preserve-status 90m $TRIPLEO_ROOT/tripleo-quickstart/quickstart.sh ${OOOQ_ARGS} 2>&1 \ + | ts '%Y-%m-%d %H:%M:%S.000 |' | sudo tee /var/log/quickstart_install.log || exit_value=2 + +# We use the tripleo.sh pingtest rather than the validate-simple role in +# quickstart-extras so that we can easily support the multinode scenario +# pingtests. We should add support for that to the validate simple role, +# and use that instead. +# https://github.com/openstack/tripleo-heat-templates/tree/master/ci/pingtests +if [[ -e ${OOO_WORKDIR_LOCAL}/overcloudrc ]]; then + $TRIPLEO_CI_DIR/tripleo-ci/scripts/tripleo.sh --overcloud-pingtest 2>&1 | sudo dd of=/var/log/overcloud-pingtest.txt +fi + +sudo journalctl -u os-collect-config | sudo tee /var/log/os-collect-config.txt + +# TODO(sshnaidm): to prepare collect-logs role for tripleo-ci specific paths +# and remove this function then +collect_oooq_logs + +# TODO(sshnaidm): fix this either in role or quickstart.sh +# it will not duplicate logs from undercloud and 127.0.0.2 +sed -i 's/hosts: all:!localhost/hosts: all:!localhost:!127.0.0.2/' $OPT_WORKDIR/playbooks/collect-logs.yml ||: + +# To collect logs from the overcloud nodes we need to use the local ssh config +export SSH_CONFIG=$OPT_WORKDIR/ssh.config.local.ansible + +# TODO(sshnaidm): to move postci functionality into collect-logs role +$TRIPLEO_ROOT/tripleo-quickstart/quickstart.sh --bootstrap --no-clone \ + $OOOQ_DEFAULT_ARGS \ + --requirements requirements.txt \ + --requirements quickstart-extras-requirements.txt \ + --config $CONFIG \ + --playbook collect-logs.yml \ + -e artcl_collect_dir=/var/log/oooq/collected_logs \ + -e @$TRIPLEO_ROOT/tripleo-ci/scripts/quickstart/multinode-settings.yml \ + -e tripleo_root=$TRIPLEO_ROOT \ + 127.0.0.2 2>&1| sudo tee /var/log/quickstart_collectlogs.log ||: + +popd + +echo 'Run completed.' +# TODO(sshnaidm): remove this when we're sure there's enough space +# Watch free space, the outage could break jobs +sudo df -h +exit $exit_value