openstack/tripleo-ci
Code Issues Proposed changes

Merge "Move toci BASH variables to ansible"

Browse Source
This commit is contained in:
Zuul 2018-09-18 21:29:25 +00:00 committed by Gerrit Code Review
commit 1ab1df6de2
9 changed files with 68 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
playbooks/tripleo-ci/templates/oooq_common_functions.sh.j2
View File

@ -114,7 +114,7 @@ function create_collect_logs_script {
export ANSIBLE_CONFIG=$OOOQ_DIR/ansible.cfg
export ARA_DATABASE=sqlite:///${LOCAL_WORKING_DIR}/ara.sqlite
export ZUUL_CHANGES=${ZUUL_CHANGES:-''}
export NODES_FILE=${NODES_FILE:-''}
export NODES_FILE={{ nodes_file }}
export TOCI_JOBTYPE=$TOCI_JOBTYPE
export STABLE_RELEASE=${STABLE_RELEASE:-''}
export QUICKSTART_RELEASE=${QUICKSTART_RELEASE:-''}

70
playbooks/tripleo-ci/templates/toci_gate_test.sh.j2
View File

@ -19,10 +19,8 @@ if [ -f /etc/nodepool/provider ] ; then
# source variables common across all the scripts.
source /etc/ci/mirror_info.sh
export RHCLOUD=''
if [ ${NODEPOOL_PROVIDER:-''} == 'rdo-cloud-tripleo' ]; then
RHCLOUD='rdocloud'
source $(dirname $0)/scripts/$RHCLOUD.env
source $(dirname $0)/scripts/rdocloud.env
# In order to save space remove the cached git repositories, at this point in
# CI the ones we are interested in have been cloned to /opt/stack/new. We
@ -58,13 +56,6 @@ export PERIODIC=0
export QUICKSTART_RELEASE="${STABLE_RELEASE:-master}"
# Stores OVB undercloud instance id
export UCINSTANCEID="null"
# Define environment variables file
export ENV_VARS=""
# Define file with set of features to test
export FEATURESET_FILE=""
export FEATURESET_CONF=""
# Define file with nodes topology
export NODES_FILE=""
# Set the number of overcloud nodes
export NODECOUNT=0
# Set the overcloud hosts for multinode
@ -74,12 +65,8 @@ export SUBNODES_SSH_KEY=
TIMEOUT_SECS=$((DEVSTACK_GATE_TIMEOUT*60))
export EXTRA_VARS=${EXTRA_VARS:-""}
export VXLAN_VARS=${VXLAN_VARS:-""}
export NODES_ARGS=""
export EXTRANODE=""
export EMIT_RELEASES_EXTRA_ARGS=""
# Set playbook execution status
export PLAYBOOK_DRY_RUN=${PLAYBOOK_DRY_RUN:=0}
export COLLECT_CONF="$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/collect-logs.yml"
LOCAL_WORKING_DIR="$WORKSPACE/.quickstart"
LWD=$LOCAL_WORKING_DIR
@ -91,49 +78,24 @@ case {{ environment_type | default('standalone') }} in
ovb)
OVB=1
UCINSTANCEID=$(http_proxy= curl http://169.254.169.254/openstack/2015-10-15/meta_data.json | python -c 'import json, sys; print json.load(sys.stdin)["uuid"]')
ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/ovb.yml"
if [[ -f "$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/ovb-$RHCLOUD.yml" ]]; then
ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/ovb-$RHCLOUD.yml"
fi
;;
multinode)
SUBNODES_SSH_KEY=/etc/nodepool/id_rsa
FEATURESET_CONF=" --extra-vars @$LWD/config/general_config/featureset-multinode-common.yml $FEATURESET_CONF"
ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode.yml"
if [[ $NODEPOOL_PROVIDER == "rdo-cloud-tripleo" ]]; then
ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode-rdocloud.yml"
fi
CONTROLLER_HOSTS=$(sed -n 1,1p /etc/nodepool/sub_nodes_private)
OVERCLOUD_HOSTS=$(cat /etc/nodepool/sub_nodes_private)
;;
singlenode)
FEATURESET_CONF=" --extra-vars @$LWD/config/general_config/featureset-multinode-common.yml $FEATURESET_CONF"
ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode.yml"
if [[ $NODEPOOL_PROVIDER == "rdo-cloud-tripleo" ]]; then
ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode-rdocloud.yml"
fi
;;
standalone)
FEATURESET_CONF=" --extra-vars @$LWD/config/general_config/featureset-multinode-common.yml $FEATURESET_CONF"
ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode.yml"
if [[ $NODEPOOL_PROVIDER == "rdo-cloud-tripleo" ]]; then
ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode-rdocloud.yml"
fi
;;
esac
EXTRA_TAGS=""
FEATURESET_FILE="{{ tripleo_root}}/tripleo-quickstart/config/general_config/featureset{{ featureset }}.yml"
FEATURESET_CONF="$FEATURESET_CONF --extra-vars @$FEATURESET_FILE"
MIXED_UPGRADE_TYPE=''
# Order matters. ffu featureset has both mixed version and ffu_overcloud_upgrade.
if is_featureset ffu_overcloud_upgrade "$FEATURESET_FILE"; then
if is_featureset ffu_overcloud_upgrade "{{ job_featureset_file }}"; then
MIXED_UPGRADE_TYPE='ffu_upgrade'
elif is_featureset mixed_upgrade "$FEATURESET_FILE"; then
elif is_featureset mixed_upgrade "{{ job_featureset_file }}"; then
MIXED_UPGRADE_TYPE='mixed_upgrade'
elif is_featureset overcloud_update "$FEATURESET_FILE"; then
elif is_featureset overcloud_update "{{ job_featureset_file }}"; then
EXTRA_TAGS=",overcloud-update"
elif is_featureset undercloud_upgrade "$FEATURESET_FILE"; then
elif is_featureset undercloud_upgrade "{{ job_featureset_file }}"; then
EXTRA_TAGS=",undercloud-upgrade"
export UPGRADE_RELEASE=$QUICKSTART_RELEASE
export QUICKSTART_RELEASE=$(previous_release_mixed_upgrade_case "${UPGRADE_RELEASE}")
@ -147,43 +109,35 @@ fi
export EXTRA_TAGS
{% if nodes is defined %}
NODES_FILE="$TRIPLEO_ROOT/tripleo-quickstart/config/nodes/{{ nodes }}.yml"
{% endif %}
{% if periodic|bool %}
PERIODIC=1
QUICKSTART_RELEASE="promotion-testing-hash-${QUICKSTART_RELEASE}"
EMIT_RELEASES_EXTRA_ARGS="$EMIT_RELEASES_EXTRA_ARGS --is-periodic"
{% endif %}
{% if dryrun|default(false)|bool %}
#TODO(rfolco): Remove PLAYBOOK_DRY_RUN when bash code is translated to ansible playbooks
PLAYBOOK_DRY_RUN=1
{% endif %}
if [[ -f "$RELEASES_SCRIPT" ]] && [[ $FEATURESET_FILE =~ '037' || $FEATURESET_FILE =~ '050' || $FEATURESET_FILE =~ '010' || $FEATURESET_FILE =~ '011' ]]; then
if [[ -f "$RELEASES_SCRIPT" ]] && [[ {{ featureset }} =~ 037|050|010|011 ]]; then
python $RELEASES_SCRIPT \
--stable-release ${STABLE_RELEASE:-"master"} \
--featureset-file $TRIPLEO_ROOT/tripleo-quickstart/config/general_config/$(basename $FEATURESET_FILE) \
--featureset-file $TRIPLEO_ROOT/tripleo-quickstart/config/general_config/$(basename {{ job_featureset_file }}) \
--output-file $RELEASES_FILE_OUTPUT \
--log-file $RELEASES_SCRIPT_LOGFILE \
$EMIT_RELEASES_EXTRA_ARGS
fi
if [[ ! -z $NODES_FILE ]]; then
{% if nodes is defined %}
if [[ ! -z {{ nodes_file }} ]]; then
pushd $TRIPLEO_ROOT/tripleo-quickstart
NODECOUNT=$(shyaml get-value node_count < $NODES_FILE)
NODECOUNT=$(shyaml get-value node_count < {{ nodes_file }})
popd
NODES_ARGS="--extra-vars @$NODES_FILE"
for PART in $(sed 's/_/ /g' <<< "$NODES_FILE") ; do
for PART in $(sed 's/_/ /g' <<< "{{ nodes }}") ; do
if [[ "$PART" == *"supp"* ]]; then
EXTRANODE=" --extra-nodes ${PART//[!0-9]/} "
fi;
done
fi
{% endif %}
# Import gated tripleo-upgrade in oooq for upgrades/updates jobs
if [[ -d $TRIPLEO_ROOT/tripleo-upgrade ]]; then

32
playbooks/tripleo-ci/templates/toci_quickstart.sh.j2
View File

@ -2,8 +2,6 @@
set -eux
set -o pipefail
export ANSIBLE_NOCOLOR=1
[[ -n ${STATS_TESTENV:-''} ]] && export STATS_TESTENV=$(( $(date +%s) - STATS_TESTENV ))
export STATS_OOOQ=$(date +%s)
LOCAL_WORKING_DIR="{{ workspace }}/.quickstart"
WORKING_DIR="$HOME"
@ -49,11 +47,11 @@ QUICKSTART_COLLECTLOGS_CMD="$LOCAL_WORKING_DIR/bin/ansible-playbook \
$LOCAL_WORKING_DIR/playbooks/collect-logs.yml \
-vv \
--extra-vars @$LOCAL_WORKING_DIR/config/release/tripleo-ci/$QUICKSTART_RELEASE.yml \
$FEATURESET_CONF \
$ENV_VARS \
{{ featureset_conf }} \
{{ env_vars }} \
{{ extra_vars }} \
$DEFAULT_ARGS \
--extra-vars @$COLLECT_CONF \
--extra-vars @{{ tripleo_root }}/tripleo-ci/toci-quickstart/config/collect-logs.yml \
--extra-vars artcl_collect_dir=$LOGS_DIR \
--tags all \
--skip-tags teardown-all \
@ -112,22 +110,21 @@ set +u
source $LOCAL_WORKING_DIR/bin/activate
set -u
source $OOOQ_DIR/ansible_ssh_env.sh
[[ -n ${STATS_OOOQ:-''} ]] && export STATS_OOOQ=$(( $(date +%s) - STATS_OOOQ ))
# Debug step capture env variables
if [[ "$PLAYBOOK_DRY_RUN" == "1" ]]; then
{% if dryrun|default(false)|bool %}
echo "-- Capture Environment Variables Used ---------"
echo "$(env)" | tee -a $LOGS_DIR/toci_env_args_output.log
declare -p | tee -a $LOGS_DIR/toci_env_args_output.log
fi
{% endif %}
echo "-- Playbooks Output --------------------------"
for playbook in {{ " ".join(playbooks) }}; do
echo "$QUICKSTART_INSTALL_CMD \
${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG} \
$NODES_ARGS \
$FEATURESET_CONF \
$ENV_VARS \
{{ nodes_args }} \
{{ featureset_conf }} \
{{ env_vars }} \
{{ extra_vars }} \
{{ vxlan_vars }} \
$DEFAULT_ARGS \
@ -138,18 +135,19 @@ for playbook in {{ " ".join(playbooks) }}; do
| tee -a $LOGS_DIR/playbook_executions.log
done
if [[ "$PLAYBOOK_DRY_RUN" == "1" ]]; then
{% if dryrun|default(false)|bool %}
# Bypassing execution, dryrun was set
exit_value=0
else
{% else %}
## LOGS COLLECTION PREPARE
create_collect_logs_script
for playbook in {{ " ".join(playbooks) }}; do
echo "${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}"
run_with_timeout $START_JOB_TIME $QUICKSTART_INSTALL_CMD \
"${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}" \
$NODES_ARGS \
$FEATURESET_CONF \
$ENV_VARS \
{{ nodes_args }} \
{{ featureset_conf }} \
{{ env_vars }} \
{{ extra_vars }} \
{{ vxlan_vars }} \
$DEFAULT_ARGS \
@ -172,7 +170,7 @@ done
fi
fi
fi
{% endif %}
popd

38
playbooks/tripleo-ci/vars/common.yaml
View File

@ -1,10 +1,38 @@
workspace: "{{ ansible_user_dir }}/workspace/"
tripleo_root: "{{ ansible_user_dir }}/src/git.openstack.org/openstack/"
workspace: "{{ ansible_user_dir }}/workspace"
tripleo_root: "{{ ansible_user_dir }}/src/git.openstack.org/openstack"
tripleo_ci_dir: "{{ ansible_user_dir }}/{{ zuul.projects['git.openstack.org/openstack-infra/tripleo-ci'].src_dir }}"
tripleo_quickstart_dir: "{{ ansible_user_dir }}/{{ zuul.projects['git.openstack.org/openstack/tripleo-quickstart'].src_dir }}"
vxlan_vars: "-e toci_vxlan_networking='false' -e vxlan_networking='false'"
extra_vars: ""
job_featureset_file: "{{ tripleo_quickstart_dir }}/config/general_config/featureset{{ featureset }}.yml"
featureset_conf: >-
{% if common_featureset is defined -%}
--extra-vars @{{ tripleo_quickstart_dir }}/config/general_config/{{ common_featureset }} {% endif -%}
--extra-vars @{{ job_featureset_file }}
periodic: >-
{%- if 'periodic' in zuul.pipeline -%}
{% if 'periodic' in zuul.pipeline -%}
true
{%- else -%}
{% else %}
false
{%- endif -%}
{%- endif %}
nodepool_provider: "{{ hostvars[inventory_hostname].nodepool.provider }}"
nodes_file: >-
{% if nodes is defined -%}
{{ tripleo_quickstart_dir }}/config/nodes/{{ nodes }}.yml
{%- endif %}
nodes_args: >-
{% if nodes is defined -%}
--extra-vars @{{ nodes_file }}
{%- endif %}
rdo_env_vars: >-
{% if nodepool_provider == 'rdo-cloud-tripleo' -%}
--extra-vars @{{ tripleo_ci_dir }}/toci-quickstart/config/testenv/{{ environment_type }}-rdocloud.yml
{%- endif %}
env_vars: "--extra-vars @{{ tripleo_ci_dir }}/toci-quickstart/config/testenv/{{ environment_type }}.yml {{ rdo_env_vars }}"

1
toci-quickstart/config/testenv/singlenode-rdocloud.yml Symbolic link
View File

@ -0,0 +1 @@
multinode-rdocloud.yml

1
toci-quickstart/config/testenv/singlenode.yml Symbolic link
View File

@ -0,0 +1 @@
multinode.yml

1
toci-quickstart/config/testenv/standalone-rdocloud.yml Symbolic link
View File

@ -0,0 +1 @@
multinode-rdocloud.yml

1
toci-quickstart/config/testenv/standalone.yml Symbolic link
View File

@ -0,0 +1 @@
multinode.yml

3
zuul.d/base.yaml
View File

@ -57,6 +57,7 @@
vars:
undercloud: 127.0.0.2
environment_infra: osinfra
common_featureset: featureset-multinode-common.yml
playbooks:
- quickstart.yml
- multinode-undercloud.yml
@ -108,6 +109,7 @@
vars:
undercloud: 127.0.0.2
environment_infra: osinfra
common_featureset: featureset-multinode-common.yml
playbooks:
- quickstart.yml
- multinode-undercloud.yml
@ -134,6 +136,7 @@
vars:
undercloud: 127.0.0.2
environment_infra: osinfra
common_featureset: featureset-multinode-common.yml
playbooks:
- quickstart.yml
- multinode-standalone.yml