CI for the TripleO project
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

toci_gate_test.sh 14KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305
  1. #!/usr/bin/env bash
  2. source $(dirname $0)/scripts/common_vars.bash
  3. source $(dirname $0)/scripts/common_functions.sh
  4. set -eux
  5. export START_JOB_TIME=$(date +%s)
  6. export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
  7. source $TRIPLEO_ROOT/tripleo-ci/scripts/oooq_common_functions.sh
  8. if [ -f /etc/nodepool/provider ] ; then
  9. # this sets
  10. # NODEPOOL_PROVIDER (e.g tripleo-test-cloud-rh1)
  11. # NODEPOOL_CLOUD (e.g.tripleo-test-cloud-rh1)
  12. # NODEPOOL_REGION (e.g. regionOne)
  13. # NODEPOOL_AZ
  14. source /etc/nodepool/provider
  15. # source variables common across all the scripts.
  16. source /etc/ci/mirror_info.sh
  17. export RHCLOUD=''
  18. if [[ ${NODEPOOL_PROVIDER:-''} == 'rdo-cloud'* ]]; then
  19. RHCLOUD='rdocloud'
  20. elif [ ${NODEPOOL_PROVIDER:-''} == 'vexxhost-rdo-ca-ymq-1' ]; then
  21. RHCLOUD='vexxhost'
  22. fi
  23. if [ -n $RHCLOUD ]; then
  24. source $(dirname $0)/scripts/$RHCLOUD.env
  25. # In order to save space remove the cached git repositories, at this point in
  26. # CI the ones we are interested in have been cloned to /opt/stack/new. We
  27. # can also remove some distro images cached on the images.
  28. sudo rm -rf /opt/git
  29. fi
  30. fi
  31. # default $NODEPOOL_PROVIDER if not already set as it's used later
  32. export NODEPOOL_PROVIDER=${NODEPOOL_PROVIDER:-""}
  33. # create logs dir (check if collect-logs doesn't already do this)
  34. mkdir -p $WORKSPACE/logs
  35. # Set job as failed until it's overwritten by pingtest/tempest real test subunit
  36. cat $TRIPLEO_ROOT/tripleo-ci/scripts/fake_fail_subunit | gzip - > $WORKSPACE/logs/testrepository.subunit.gz
  37. # NOTE(trown): In openstack-infra we have pip already, but this will ensure we
  38. # have it available in other environments.
  39. command -v pip || \
  40. (curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py"; sudo python get-pip.py)
  41. sudo yum -y install python-requests python-urllib3
  42. sudo pip install shyaml
  43. # Sets whether or not this job will upload images.
  44. export PERIODIC=0
  45. # Sets which repositories to use in the job
  46. export QUICKSTART_RELEASE="${STABLE_RELEASE:-master}"
  47. # Stores OVB undercloud instance id
  48. export UCINSTANCEID="null"
  49. # Define environment variables file
  50. export ENV_VARS=""
  51. # Define file with set of features to test
  52. export FEATURESET_FILE=""
  53. export FEATURESET_CONF=""
  54. # Define file with nodes topology
  55. export NODES_FILE=""
  56. # Set the number of overcloud nodes
  57. export NODECOUNT=0
  58. # Sets the undercloud hostname
  59. export UNDERCLOUD=""
  60. # Select the tags to run
  61. export TAGS=all
  62. # Identify in which environment we're deploying
  63. export ENVIRONMENT=""
  64. # Set the overcloud hosts for multinode
  65. export OVERCLOUD_HOSTS=
  66. export CONTROLLER_HOSTS=
  67. export SUBNODES_SSH_KEY=
  68. TIMEOUT_SECS=$((DEVSTACK_GATE_TIMEOUT*60))
  69. export EXTRA_VARS=${EXTRA_VARS:-""}
  70. export VXLAN_VARS=${VXLAN_VARS:-""}
  71. export NODES_ARGS=""
  72. export EXTRANODE=""
  73. export EMIT_RELEASES_EXTRA_ARGS=""
  74. # Set playbook execution status
  75. export PLAYBOOK_DRY_RUN=${PLAYBOOK_DRY_RUN:=0}
  76. export COLLECT_CONF="$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/collect-logs.yml"
  77. LOCAL_WORKING_DIR="$WORKSPACE/.quickstart"
  78. LWD=$LOCAL_WORKING_DIR
  79. QUICKSTART_SH_JOBS="ovb-3ctlr_1comp-featureset001 multinode-1ctlr-featureset010"
  80. export RELEASES_FILE_OUTPUT=$WORKSPACE/logs/releases.sh
  81. export RELEASES_SCRIPT=$TRIPLEO_ROOT/tripleo-ci/scripts/emit_releases_file/emit_releases_file.py
  82. export RELEASES_SCRIPT_LOGFILE=$WORKSPACE/logs/emit_releases_file.log
  83. # Assemble quickstart configuration based on job type keywords
  84. for JOB_TYPE_PART in $(sed 's/-/ /g' <<< "${TOCI_JOBTYPE:-}") ; do
  85. case $JOB_TYPE_PART in
  86. featureset*)
  87. FEATURESET_FILE="$LWD/config/general_config/$JOB_TYPE_PART.yml"
  88. # featurset_file is not yet in its final destination so we
  89. # have to use current_featureset_file.
  90. CURRENT_FEATURESET_FILE="$TRIPLEO_ROOT/tripleo-quickstart/config/general_config/$JOB_TYPE_PART.yml"
  91. FEATURESET_CONF="$FEATURESET_CONF --extra-vars @$FEATURESET_FILE"
  92. MIXED_UPGRADE_TYPE=''
  93. # Order matters. ffu featureset has both mixed version and ffu_overcloud_upgrade.
  94. if is_featureset ffu_overcloud_upgrade "${CURRENT_FEATURESET_FILE}"; then
  95. MIXED_UPGRADE_TYPE='ffu_upgrade'
  96. elif is_featureset mixed_upgrade "${CURRENT_FEATURESET_FILE}"; then
  97. MIXED_UPGRADE_TYPE='mixed_upgrade'
  98. elif is_featureset overcloud_update "${CURRENT_FEATURESET_FILE}"; then
  99. TAGS="$TAGS,overcloud-update"
  100. elif is_featureset undercloud_upgrade "${CURRENT_FEATURESET_FILE}"; then
  101. TAGS="$TAGS,undercloud-upgrade"
  102. export UPGRADE_RELEASE=$QUICKSTART_RELEASE
  103. export QUICKSTART_RELEASE=$(previous_release_mixed_upgrade_case "${UPGRADE_RELEASE}")
  104. fi
  105. # The case is iterating over TOCI_JOBTYPE which is
  106. # standalone-featureset. So featureset comes after and we
  107. # can override TAGS safely.
  108. if is_featureset standalone_upgrade "${CURRENT_FEATURESET_FILE}" ; then
  109. # We don't want "build" as it would wrongly build test
  110. # package under the N-1 version.
  111. TAGS="standalone,standalone-upgrade"
  112. fi
  113. # Set UPGRADE_RELEASE if applicable
  114. if [ -n "${MIXED_UPGRADE_TYPE}" ]; then
  115. export UPGRADE_RELEASE=$(previous_release_from "${STABLE_RELEASE}" "${MIXED_UPGRADE_TYPE}")
  116. QUICKSTART_RELEASE="$QUICKSTART_RELEASE-undercloud-$UPGRADE_RELEASE-overcloud"
  117. # Run overcloud-upgrade tag only in upgrades jobs
  118. TAGS="$TAGS,overcloud-upgrade"
  119. fi
  120. ;;
  121. ovb)
  122. OVB=1
  123. ENVIRONMENT="ovb"
  124. METADATA_FILENAME='/mnt/config/openstack/latest/meta_data.json'
  125. if sudo test -f $METADATA_FILENAME; then
  126. METADATA=$(sudo cat /mnt/config/openstack/latest/meta_data.json)
  127. set +x
  128. UCINSTANCEID=$(echo $METADATA | python -c 'import json, sys; print json.load(sys.stdin)["uuid"]')
  129. set -x
  130. else
  131. UCINSTANCEID=$(http_proxy= curl http://169.254.169.254/openstack/2015-10-15/meta_data.json | python -c 'import json, sys; print json.load(sys.stdin)["uuid"]')
  132. fi
  133. if [[ " $QUICKSTART_SH_JOBS " =~ " $TOCI_JOBTYPE " ]]; then
  134. export PLAYBOOKS=${PLAYBOOKS:-"baremetal-full-deploy.yml"}
  135. else
  136. export PLAYBOOKS=${PLAYBOOKS:-"ovb-setup.yml baremetal-full-undercloud.yml baremetal-full-overcloud-prep.yml baremetal-full-overcloud.yml baremetal-full-overcloud-validate.yml browbeat-minimal.yml"}
  137. fi
  138. ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/ovb.yml"
  139. if [[ -f "$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/ovb-$RHCLOUD.yml" ]]; then
  140. ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/ovb-$RHCLOUD.yml"
  141. fi
  142. UNDERCLOUD="undercloud"
  143. ;;
  144. multinode)
  145. SUBNODES_SSH_KEY=/etc/nodepool/id_rsa
  146. ENVIRONMENT="osinfra"
  147. if [[ " $QUICKSTART_SH_JOBS " =~ " $TOCI_JOBTYPE " ]]; then
  148. export PLAYBOOKS=${PLAYBOOKS:-"multinode.yml"}
  149. else
  150. export PLAYBOOKS=${PLAYBOOKS:-"quickstart.yml multinode-undercloud.yml multinode-overcloud-prep.yml multinode-overcloud.yml multinode-overcloud-update.yml multinode-overcloud-upgrade.yml multinode-validate.yml"}
  151. fi
  152. FEATURESET_CONF=" --extra-vars @$LWD/config/general_config/featureset-multinode-common.yml $FEATURESET_CONF"
  153. ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode.yml"
  154. if [[ $NODEPOOL_PROVIDER == "rdo-cloud"* ]]; then
  155. ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode-rdocloud.yml"
  156. fi
  157. UNDERCLOUD="127.0.0.2"
  158. TAGS="build,undercloud-setup,undercloud-scripts,undercloud-install,undercloud-post-install,tripleo-validations,overcloud-scripts,overcloud-prep-config,overcloud-prep-containers,overcloud-deploy,overcloud-post-deploy,overcloud-validate"
  159. CONTROLLER_HOSTS=$(sed -n 1,1p /etc/nodepool/sub_nodes_private)
  160. OVERCLOUD_HOSTS=$(cat /etc/nodepool/sub_nodes_private)
  161. ;;
  162. singlenode)
  163. ENVIRONMENT="osinfra"
  164. UNDERCLOUD="127.0.0.2"
  165. if [[ " $QUICKSTART_SH_JOBS " =~ " $TOCI_JOBTYPE " ]]; then
  166. export PLAYBOOKS=${PLAYBOOKS:-"multinode.yml"}
  167. else
  168. export PLAYBOOKS=${PLAYBOOKS:-"quickstart.yml multinode-undercloud.yml multinode-undercloud-upgrade.yml multinode-overcloud-prep.yml multinode-overcloud.yml multinode-overcloud-upgrade.yml multinode-validate.yml"}
  169. fi
  170. FEATURESET_CONF=" --extra-vars @$LWD/config/general_config/featureset-multinode-common.yml $FEATURESET_CONF"
  171. ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode.yml"
  172. if [[ $NODEPOOL_PROVIDER == "rdo-cloud"* ]]; then
  173. ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode-rdocloud.yml"
  174. fi
  175. TAGS="build,undercloud-setup,undercloud-scripts,undercloud-install,undercloud-validate,images"
  176. ;;
  177. standalone)
  178. ENVIRONMENT="osinfra"
  179. UNDERCLOUD="127.0.0.2"
  180. # Adding upgrade playbook here to be consistant with the v3 definition.
  181. export PLAYBOOKS=${PLAYBOOKS:-"quickstart.yml multinode-standalone.yml multinode-standalone-upgrade.yml "}
  182. FEATURESET_CONF=" --extra-vars @$LWD/config/general_config/featureset-multinode-common.yml $FEATURESET_CONF"
  183. ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode.yml"
  184. if [[ $NODEPOOL_PROVIDER == "rdo-cloud"* ]]; then
  185. ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode-rdocloud.yml"
  186. fi
  187. TAGS="build,standalone"
  188. ;;
  189. periodic)
  190. PERIODIC=1
  191. QUICKSTART_RELEASE="promotion-testing-hash-${QUICKSTART_RELEASE}"
  192. EMIT_RELEASES_EXTRA_ARGS="$EMIT_RELEASES_EXTRA_ARGS --is-periodic"
  193. ;;
  194. gate)
  195. ;;
  196. dryrun)
  197. PLAYBOOK_DRY_RUN=1
  198. ;;
  199. *)
  200. # the rest should be node configuration
  201. NODES_FILE="$TRIPLEO_ROOT/tripleo-quickstart/config/nodes/$JOB_TYPE_PART.yml"
  202. ;;
  203. esac
  204. done
  205. if [[ -f "$RELEASES_SCRIPT" ]] && [[ $FEATURESET_FILE =~ 010|011|037|047|050|056 ]]; then
  206. python $RELEASES_SCRIPT \
  207. --stable-release ${STABLE_RELEASE:-"master"} \
  208. --featureset-file $TRIPLEO_ROOT/tripleo-quickstart/config/general_config/$(basename $FEATURESET_FILE) \
  209. --output-file $RELEASES_FILE_OUTPUT \
  210. --log-file $RELEASES_SCRIPT_LOGFILE \
  211. $EMIT_RELEASES_EXTRA_ARGS
  212. fi
  213. if [[ ! -z $NODES_FILE ]]; then
  214. pushd $TRIPLEO_ROOT/tripleo-quickstart
  215. NODECOUNT=$(shyaml get-value node_count < $NODES_FILE)
  216. popd
  217. NODES_ARGS="--extra-vars @$NODES_FILE"
  218. for PART in $(sed 's/_/ /g' <<< "$NODES_FILE") ; do
  219. if [[ "$PART" == *"supp"* ]]; then
  220. EXTRANODE=" --extra-nodes ${PART//[!0-9]/} "
  221. fi;
  222. done
  223. fi
  224. # Import gated tripleo-upgrade in oooq for upgrades/updates jobs
  225. if [[ -d $TRIPLEO_ROOT/tripleo-upgrade ]]; then
  226. echo "file://${TRIPLEO_ROOT}/tripleo-upgrade/#egg=tripleo-upgrade" >> ${TRIPLEO_ROOT}/tripleo-quickstart/quickstart-extras-requirements.txt
  227. else
  228. # Otherwise, if not importing it, oooq will fail when loading
  229. # tripleo-upgrade role in the playbook.
  230. echo "git+https://opendev.org/openstack/tripleo-upgrade.git@${ZUUL_BRANCH}#egg=tripleo-upgrade" >> ${TRIPLEO_ROOT}/tripleo-quickstart/quickstart-extras-requirements.txt
  231. fi
  232. # Import gated external repo in oooq
  233. for EXTERNAL_REPO in 'browbeat' 'tripleo-ha-utils' 'tripleo-quickstart-extras'; do
  234. if [[ -d $TRIPLEO_ROOT/$EXTERNAL_REPO ]]; then
  235. sed -i "s#git+https://opendev.org/openstack/$EXTERNAL_REPO#file://${TRIPLEO_ROOT}/$EXTERNAL_REPO#1" ${TRIPLEO_ROOT}/tripleo-quickstart/quickstart-extras-requirements.txt
  236. fi
  237. done
  238. # Start time tracking
  239. export STATS_TESTENV=$(date +%s)
  240. pushd $TRIPLEO_ROOT/tripleo-ci
  241. if [ -e $WORKSPACE/instackenv.json -a "$ENVIRONMENT" = "ovb" ] ; then
  242. echo "Running without te-broker"
  243. export TE_DATAFILE=$WORKSPACE/instackenv.json
  244. ./toci_quickstart.sh
  245. elif [ "$ENVIRONMENT" = "ovb" ] ; then
  246. # We only support multi-nic at the moment
  247. NETISO_ENV="multi-nic"
  248. ./toci_quickstart.sh
  249. else
  250. # Copy nodepool keys to current user
  251. sudo cp /etc/nodepool/id_rsa* $HOME/.ssh/
  252. sudo chown $USER:$USER $HOME/.ssh/id_rsa*
  253. chmod 0600 $HOME/.ssh/id_rsa*
  254. cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys
  255. # pre-ansible requirement
  256. sudo mkdir -p /root/.ssh/
  257. # everything below here *MUST* be translated to a role ASAP
  258. # empty image to fool overcloud deployment
  259. # set no_proxy variable
  260. export IP_DEVICE=${IP_DEVICE:-"eth0"}
  261. MY_IP=$(ip addr show dev $IP_DEVICE | awk '/inet / {gsub("/.*", "") ; print $2}')
  262. MY_IP_eth1=$(ip addr show dev eth1 | awk '/inet / {gsub("/.*", "") ; print $2}') || MY_IP_eth1=""
  263. export http_proxy=""
  264. undercloud_net_range="192.168.24."
  265. undercloud_services_ip=$undercloud_net_range"1"
  266. undercloud_haproxy_public_ip=$undercloud_net_range"2"
  267. undercloud_haproxy_admin_ip=$undercloud_net_range"3"
  268. export no_proxy=$undercloud_services_ip,$undercloud_haproxy_public_ip,$undercloud_haproxy_admin_ip,$MY_IP,$MY_IP_eth1
  269. # finally, run quickstart
  270. ./toci_quickstart.sh
  271. fi
  272. echo "Run completed"
  273. echo "tripleo.${STABLE_RELEASE:-master}.${TOCI_JOBTYPE}.logs.size_mb" "$(du -sm $WORKSPACE/logs | awk {'print $1'})" "$(date +%s)" | nc 66.187.229.172 2003 || true