CI for the TripleO project
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

tripleo.sh 63KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628
  1. #!/bin/bash
  2. # Copyright 2015 Red Hat, Inc.
  3. # All Rights Reserved.
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  6. # not use this file except in compliance with the License. You may obtain
  7. # a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  13. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  14. # License for the specific language governing permissions and limitations
  15. # under the License.
  16. ##############################################################################
  17. # tripleo.sh is a script to automate a TripleO setup. It's goals are to be
  18. # used in aiding:
  19. #
  20. # - developer setups
  21. # - CI
  22. # - documentation generation (hopefully)
  23. #
  24. # It's not a new CLI, or end user facing wrapper around existing TripleO
  25. # CLI's.
  26. #
  27. # tripleo.sh should never contain any "business" logic in it that is
  28. # necessary for a successful deployment. It should instead just mirror the
  29. # steps that we document for TripleO end users.
  30. #
  31. ##############################################################################
  32. set -eu
  33. set -o pipefail
  34. SCRIPT_NAME=${SCRIPT_NAME:-$(basename $0)}
  35. function show_options {
  36. echo "Usage: $SCRIPT_NAME [options]"
  37. echo
  38. echo "Automates TripleO setup steps."
  39. echo
  40. echo "$SCRIPT_NAME is also configurable via environment variables, most of"
  41. echo "which are not exposed via cli args for simplicity. See the source"
  42. echo "for the set of environment variables that can be overridden."
  43. echo
  44. echo "Note that cli args always take precedence over environment"
  45. echo "variables."
  46. echo
  47. echo "Options:"
  48. echo " --repo-setup -- Perform repository setup."
  49. echo " --delorean-setup -- Install local delorean build environment."
  50. echo " --delorean-build -- Build a delorean package locally"
  51. echo " --multinode-setup -- Perform multinode setup."
  52. echo " --bootstrap-subnodes -- Perform bootstrap setup on subnodes. WARNING bootstrap-subnodes is deprecated and will be removed."
  53. echo " --setup-nodepool-files -- Setup nodepool files on subnodes."
  54. echo " --undercloud -- Install the undercloud."
  55. echo " --overcloud-images -- Build and load overcloud images."
  56. echo " --register-nodes -- Register and configure nodes."
  57. echo " --introspect-nodes -- Introspect nodes."
  58. echo " --undercloud-upgrade -- Upgrade a deployed undercloud."
  59. echo " --overcloud-deploy -- Deploy an overcloud."
  60. echo " --overcloud-update -- Update a deployed overcloud."
  61. echo " --overcloud-upgrade -- Upgrade a deployed overcloud."
  62. echo " --overcloud-upgrade-converge -- Finish (converge) upgrade of a deployed overcloud."
  63. echo " --overcloud-delete -- Delete the overcloud."
  64. echo " --use-containers -- Use a containerized compute node."
  65. echo " --enable-check -- Enable checks on update."
  66. echo " --overcloud-pingtest -- Run a tenant vm, attach and ping floating IP."
  67. echo " --overcloud-sanitytest -- Run some basic crud checks for each service."
  68. echo " --skip-sanitytest-create -- Do not create resources when performing a sanitytest (assume they exist)."
  69. echo " --skip-sanitytest-cleanup -- Do not delete the created resources when performing a sanitytest."
  70. echo " --skip-pingtest-cleanup -- For debuging purposes, do not delete the created resources when performing a pingtest."
  71. echo " --run-tempest -- Run tempest tests."
  72. echo " --all, -a -- Run all of the above commands."
  73. echo " -x -- enable tracing"
  74. echo " --help, -h -- Print this help message."
  75. echo
  76. exit 1
  77. }
  78. if [ ${#@} = 0 ]; then
  79. show_options
  80. exit 1
  81. fi
  82. TEMP=$(getopt -o ,h \
  83. -l,help,repo-setup,delorean-setup,delorean-build,multinode-setup,bootstrap-subnodes,undercloud,overcloud-images,register-nodes,introspect-nodes,overcloud-deploy,overcloud-update,overcloud-upgrade,overcloud-upgrade-converge,overcloud-delete,use-containers,overcloud-pingtest,undercloud-upgrade,skip-pingtest-cleanup,all,enable-check,run-tempest,setup-nodepool-files,overcloud-sanitytest,skip-sanitytest-create,skip-sanitytest-cleanup \
  84. -o,x,h,a \
  85. -n $SCRIPT_NAME -- "$@")
  86. if [ $? != 0 ]; then
  87. show_options
  88. exit 1
  89. fi
  90. # Note the quotes around `$TEMP': they are essential!
  91. eval set -- "$TEMP"
  92. TRIPLEO_ROOT=${TRIPLEO_ROOT:-$HOME/tripleo}
  93. # Source deploy.env if it exists. It should exist if we are running under
  94. # tripleo-ci
  95. if [ -f "$TRIPLEO_ROOT/tripleo-ci/deploy.env" ]; then
  96. source $TRIPLEO_ROOT/tripleo-ci/deploy.env
  97. fi
  98. ALL=${ALL:-""}
  99. TRIPLEO_HEAT_TEMPLATES_ROOT=${TRIPLEO_HEAT_TEMPLATES_ROOT:-"/usr/share/openstack-tripleo-heat-templates"}
  100. CONTAINER_ARGS=${CONTAINER_ARGS:-"-e ${TRIPLEO_HEAT_TEMPLATES_ROOT}/environments/docker.yaml --libvirt-type=qemu"}
  101. STABLE_RELEASE=${STABLE_RELEASE:-}
  102. REVIEW_RELEASE=${REVIEW_RELEASE:-}
  103. UPGRADE_RELEASE=${UPGRADE_RELEASE:-""}
  104. DELOREAN_REPO_FILE=${DELOREAN_REPO_FILE:-"delorean.repo"}
  105. DELOREAN_REPO_URL=${DELOREAN_REPO_URL:-"\
  106. https://trunk.rdoproject.org/centos7/current-tripleo/"}
  107. DELOREAN_STABLE_REPO_URL=${DELOREAN_STABLE_REPO_URL:-"\
  108. https://trunk.rdoproject.org/centos7-$STABLE_RELEASE/current/"}
  109. ATOMIC_URL=${ATOMIC_URL:-"https://download.fedoraproject.org/pub/alt/atomic/stable/Cloud-Images/x86_64/Images/Fedora-Cloud-Atomic-23-20160308.x86_64.qcow2"}
  110. INSTACKENV_JSON_PATH=${INSTACKENV_JSON_PATH:-"$HOME/instackenv.json"}
  111. INTROSPECT_NODES=${INTROSPECT_NODES:-""}
  112. REGISTER_NODES=${REGISTER_NODES:-""}
  113. OVERCLOUD_DEPLOY=${OVERCLOUD_DEPLOY:-""}
  114. OVERCLOUD_DELETE=${OVERCLOUD_DELETE:-""}
  115. OVERCLOUD_DELETE_TIMEOUT=${OVERCLOUD_DELETE_TIMEOUT:-"300"}
  116. OVERCLOUD_DEPLOY_ARGS=${OVERCLOUD_DEPLOY_ARGS:-""}
  117. OVERCLOUD_VALIDATE_ARGS=${OVERCLOUD_VALIDATE_ARGS-"--validation-warnings-fatal"}
  118. OVERCLOUD_UPDATE=${OVERCLOUD_UPDATE:-""}
  119. OVERCLOUD_UPGRADE=${OVERCLOUD_UPGRADE:-""}
  120. OVERCLOUD_UPGRADE_CONVERGE=${OVERCLOUD_UPGRADE_CONVERGE:-""}
  121. OVERCLOUD_UPDATE_RM_FILES=${OVERCLOUD_UPDATE_RM_FILES:-"1"}
  122. OVERCLOUD_UPDATE_ARGS=${OVERCLOUD_UPDATE_ARGS:-"$OVERCLOUD_DEPLOY_ARGS $OVERCLOUD_VALIDATE_ARGS"}
  123. OVERCLOUD_UPDATE_CHECK=${OVERCLOUD_UPDATE_CHECK:-}
  124. OVERCLOUD_IMAGES_PATH=${OVERCLOUD_IMAGES_PATH:-"$HOME"}
  125. OVERCLOUD_IMAGES_YAML_PATH=${OVERCLOUD_IMAGES_YAML_PATH:-"/usr/share/openstack-tripleo-common/image-yaml"}
  126. OVERCLOUD_IMAGES=${OVERCLOUD_IMAGES:-""}
  127. OVERCLOUD_IMAGES_LEGACY_ARGS=${OVERCLOUD_IMAGES_LEGACY_ARGS:-"--all"}
  128. OVERCLOUD_IMAGES_ARGS=${OVERCLOUD_IMAGES_ARGS:-"--output-directory $OVERCLOUD_IMAGES_PATH --config-file $OVERCLOUD_IMAGES_YAML_PATH/overcloud-images.yaml --config-file $OVERCLOUD_IMAGES_YAML_PATH/overcloud-images-centos7.yaml"}
  129. OVERCLOUD_NAME=${OVERCLOUD_NAME:-"overcloud"}
  130. OVERCLOUD_UPGRADE_THT_PATH=${OVERCLOUD_UPGRADE_THT_PATH:-"/usr/share/openstack-tripleo-heat-templates"}
  131. OVERCLOUD_UPGRADE_ARGS=${OVERCLOUD_UPGRADE_ARGS:-"-e $OVERCLOUD_UPGRADE_THT_PATH/overcloud-resource-registry-puppet.yaml $OVERCLOUD_DEPLOY_ARGS -e $OVERCLOUD_UPGRADE_THT_PATH/environments/major-upgrade-composable-steps.yaml -e $HOME/init-repo.yaml --templates $OVERCLOUD_UPGRADE_THT_PATH"}
  132. OVERCLOUD_UPGRADE_CONVERGE_ARGS=${OVERCLOUD_UPGRADE_CONVERGE_ARGS:-"-e $OVERCLOUD_UPGRADE_THT_PATH/overcloud-resource-registry-puppet.yaml $OVERCLOUD_DEPLOY_ARGS -e $OVERCLOUD_UPGRADE_THT_PATH/environments/major-upgrade-converge.yaml --templates $OVERCLOUD_UPGRADE_THT_PATH"}
  133. UNDERCLOUD_UPGRADE=${UNDERCLOUD_UPGRADE:-""}
  134. OVERCLOUD_SANITYTEST_SKIP_CREATE=${OVERCLOUD_SANITYTEST_SKIP_CREATE:-""}
  135. OVERCLOUD_SANITYTEST_SKIP_CLEANUP=${OVERCLOUD_SANITYTEST_SKIP_CLEANUP:-""}
  136. OVERCLOUD_SANITYTEST=${OVERCLOUD_SANITYTEST:-""}
  137. SANITYTEST_CONTENT_NAME=${SANITYTEST_CONTENT_NAME:-"sanity_test"}
  138. SKIP_PINGTEST_CLEANUP=${SKIP_PINGTEST_CLEANUP:-""}
  139. OVERCLOUD_PINGTEST=${OVERCLOUD_PINGTEST:-""}
  140. UNDERCLOUD_SANITY_CHECK=${UNDERCLOUD_SANITY_CHECK:-""}
  141. REPO_SETUP=${REPO_SETUP:-""}
  142. REPO_PREFIX=${REPO_PREFIX:-"/etc/yum.repos.d/"}
  143. CACHEUPLOAD=${CACHEUPLOAD:-"0"}
  144. OVERCLOUD_IMAGES_DIB_YUM_REPO_CONF=${OVERCLOUD_IMAGES_DIB_YUM_REPO_CONF:-"\
  145. $REPO_PREFIX/delorean.repo \
  146. $REPO_PREFIX/delorean-current.repo \
  147. $REPO_PREFIX/delorean-deps.repo"}
  148. CEPH_RELEASE=jewel
  149. CEPH_REPO_FILE=centos-ceph-$CEPH_RELEASE.repo
  150. if [[ -e /etc/ci/mirror_info.sh ]]; then
  151. source /etc/ci/mirror_info.sh
  152. fi
  153. NODEPOOL_CENTOS_MIRROR=${NODEPOOL_CENTOS_MIRROR:-http://mirror.centos.org/centos}
  154. NODEPOOL_RDO_PROXY=${NODEPOOL_RDO_PROXY:-https://trunk.rdoproject.org}
  155. NODEPOOL_BUILDLOGS_CENTOS_PROXY="${NODEPOOL_BUILDLOGS_CENTOS_PROXY:-https://buildlogs.centos.org}"
  156. NODEPOOL_CBS_CENTOS_PROXY="${NODEPOOL_CBS_CENTOS_PROXY:-https://cbs.centos.org/repos}"
  157. OVERCLOUD_IMAGES_DIB_YUM_REPO_CONF=${OVERCLOUD_IMAGES_DIB_YUM_REPO_CONF}"\
  158. $REPO_PREFIX/$CEPH_REPO_FILE"
  159. OPSTOOLS_REPO_ENABLED=${OPSTOOLS_REPO_ENABLED:-"0"}
  160. if [[ "${OPSTOOLS_REPO_ENABLED}" = 1 ]]; then
  161. OVERCLOUD_IMAGES_DIB_YUM_REPO_CONF=${OVERCLOUD_IMAGES_DIB_YUM_REPO_CONF}"\
  162. $REPO_PREFIX/centos-opstools.repo"
  163. fi
  164. FEATURE_BRANCH=${FEATURE_BRANCH:-}
  165. DELOREAN_SETUP=${DELOREAN_SETUP:-""}
  166. DELOREAN_BUILD=${DELOREAN_BUILD:-""}
  167. MULTINODE_SETUP=${MULTINODE_SETUP:-""}
  168. MULTINODE_ENV_NAME=${MULTINODE_ENV_NAME:-}
  169. MTU=${MTU:-"1450"}
  170. BOOTSTRAP_SUBNODES=${BOOTSTRAP_SUBNODES:-""}
  171. SETUP_NODEPOOL_FILES=${SETUP_NODEPOOL_FILES:-""}
  172. PRIMARY_NODE_IP=${PRIMARY_NODE_IP:-""}
  173. SUB_NODE_IPS=${SUB_NODE_IPS:-""}
  174. NODEPOOL_REGION=${NODEPOOL_REGION:-"nodepool_region"}
  175. NODEPOOL_CLOUD=${NODEPOOL_CLOUD:-"nodepool_cloud"}
  176. STDERR=/dev/null
  177. UNDERCLOUD=${UNDERCLOUD:-""}
  178. UNDERCLOUD_CONF=${UNDERCLOUD_CONF:-"/usr/share/instack-undercloud/undercloud.conf.sample"}
  179. UNDERCLOUD_SSL=${UNDERCLOUD_SSL:-""}
  180. BASE=${BASE:-$TRIPLEO_ROOT}
  181. USE_CONTAINERS=${USE_CONTAINERS:-""}
  182. TEMPEST_RUN=${TEMPEST_RUN:-""}
  183. TEMPEST_ARGS=${TEMPEST_ARGS:-"--parallel --subunit"}
  184. TEMPEST_ADD_CONFIG=${TEMPEST_ADD_CONFIG:-}
  185. TEMPEST_REGEX=${TEMPEST_REGEX:-"^(?=(.*smoke))(?!(tempest.api.orchestration.stacks|tempest.scenario.test_volume_boot_pattern|tempest.api.telemetry))"}
  186. TEMPEST_PINNED="72ccabcb685df7c3e28cd25639b05d8a031901c8"
  187. SSH_OPTIONS=${SSH_OPTIONS:-'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=Verbose -o PasswordAuthentication=no -o ConnectionAttempts=32'}
  188. ALT_OVERCLOUDRC=${ALT_OVERCLOUDRC:-""}
  189. export SCRIPTS_DIR=$(dirname ${BASH_SOURCE[0]:-$0})
  190. OVB=${OVB:-0}
  191. # Make sure we use Puppet to deploy packages on scenario upgrades jobs after ocata release
  192. if [[ "${STABLE_RELEASE}" != "newton" ]] ; then
  193. OVERCLOUD_UPGRADE_ARGS="$OVERCLOUD_UPGRADE_ARGS -e $TRIPLEO_ROOT/tripleo-ci/test-environments/enable_package_install.yaml "
  194. OVERCLOUD_UPGRADE_CONVERGE_ARGS="$OVERCLOUD_UPGRADE_CONVERGE_ARGS -e $TRIPLEO_ROOT/tripleo-ci/test-environments/enable_package_install.yaml "
  195. fi
  196. # Temporary workarounds
  197. while true ; do
  198. case "$1" in
  199. --all|-a ) ALL="1"; shift 1;;
  200. --use-containers) USE_CONTAINERS="1"; shift 1;;
  201. --enable-check) OVERCLOUD_UPDATE_CHECK="1"; shift 1;;
  202. --introspect-nodes) INTROSPECT_NODES="1"; shift 1;;
  203. --register-nodes) REGISTER_NODES="1"; shift 1;;
  204. --overcloud-deploy) OVERCLOUD_DEPLOY="1"; shift 1;;
  205. --overcloud-update) OVERCLOUD_UPDATE="1"; shift 1;;
  206. --overcloud-upgrade) OVERCLOUD_UPGRADE="1"; shift 1;;
  207. --overcloud-upgrade-converge) OVERCLOUD_UPGRADE_CONVERGE="1"; shift 1;;
  208. --overcloud-delete) OVERCLOUD_DELETE="1"; shift 1;;
  209. --overcloud-images) OVERCLOUD_IMAGES="1"; shift 1;;
  210. --overcloud-pingtest) OVERCLOUD_PINGTEST="1"; shift 1;;
  211. --skip-pingtest-cleanup) SKIP_PINGTEST_CLEANUP="1"; shift 1;;
  212. --overcloud-sanitytest) OVERCLOUD_SANITYTEST="1"; shift 1;;
  213. --skip-sanitytest-create) OVERCLOUD_SANITYTEST_SKIP_CREATE="1"; shift 1;;
  214. --skip-sanitytest-cleanup) OVERCLOUD_SANITYTEST_SKIP_CLEANUP="1"; shift 1;;
  215. --run-tempest) TEMPEST_RUN="1"; shift 1;;
  216. --repo-setup) REPO_SETUP="1"; shift 1;;
  217. --delorean-setup) DELOREAN_SETUP="1"; shift 1;;
  218. --delorean-build) DELOREAN_BUILD="1"; shift 1;;
  219. --undercloud) UNDERCLOUD="1"; shift 1;;
  220. --undercloud-upgrade) UNDERCLOUD_UPGRADE="1"; shift 1;;
  221. --multinode-setup) MULTINODE_SETUP="1"; shift 1;;
  222. --bootstrap-subnodes) BOOTSTRAP_SUBNODES="1"; shift 1;;
  223. --setup-nodepool-files) SETUP_NODEPOOL_FILES="1"; shift 1;;
  224. -x) set -x; STDERR=/dev/stderr; shift 1;;
  225. -h | --help) show_options 0;;
  226. --) shift ; break ;;
  227. *) echo "Error: unsupported option $1." ; exit 1 ;;
  228. esac
  229. done
  230. function log {
  231. echo "#################"
  232. echo -n "$SCRIPT_NAME -- "
  233. echo $@
  234. echo "#################"
  235. }
  236. function source_rc {
  237. if [ $1 = "stackrc" ]; then
  238. cloud="Undercloud"
  239. else
  240. cloud="Overcloud"
  241. fi
  242. echo "You must source a $1 file for the $cloud."
  243. echo "Attempting to source $HOME/$1"
  244. source $HOME/$1
  245. echo "Done"
  246. }
  247. function stackrc_check {
  248. source_rc "stackrc"
  249. }
  250. function overcloudrc_check {
  251. if [ -z "$ALT_OVERCLOUDRC" ]; then
  252. source_rc "overcloudrc"
  253. else
  254. source_rc "$ALT_OVERCLOUDRC"
  255. fi
  256. }
  257. function repo_setup {
  258. log "Repository setup"
  259. sudo yum clean metadata
  260. # sets $TRIPLEO_OS_FAMILY and $TRIPLEO_OS_DISTRO
  261. source $(dirname ${BASH_SOURCE[0]:-$0})/set-os-type
  262. if [ "$TRIPLEO_OS_DISTRO" = "centos" ]; then
  263. # Enable Storage/SIG Ceph repo
  264. if rpm -q centos-release-ceph-jewel; then
  265. sudo yum -y erase centos-release-ceph-jewel
  266. fi
  267. sudo /bin/bash -c "cat <<-EOF>$REPO_PREFIX/$CEPH_REPO_FILE
  268. [centos-ceph-$CEPH_RELEASE]
  269. name=centos-ceph-$CEPH_RELEASE
  270. baseurl=$NODEPOOL_CENTOS_MIRROR/7/storage/x86_64/ceph-$CEPH_RELEASE/
  271. gpgcheck=0
  272. enabled=1
  273. EOF"
  274. if [[ "${OPSTOOLS_REPO_ENABLED}" = 1 ]]; then
  275. sudo /bin/bash -c "cat <<-EOF>$REPO_PREFIX/centos-opstools.repo
  276. [centos-opstools]
  277. name=centos-opstools
  278. baseurl=$NODEPOOL_CENTOS_MIRROR/7/opstools/x86_64/
  279. gpgcheck=0
  280. enabled=1
  281. EOF"
  282. fi
  283. fi
  284. # @matbu TBR debuginfo:
  285. log "Stable release: $STABLE_RELEASE"
  286. if [ -z "$STABLE_RELEASE" ]; then
  287. # Enable the Delorean Deps repository
  288. sudo curl -fLvo $REPO_PREFIX/delorean-deps.repo https://trunk.rdoproject.org/centos7/delorean-deps.repo
  289. sudo sed -i -e 's%priority=.*%priority=30%' $REPO_PREFIX/delorean-deps.repo
  290. sudo sed -i -e "s~http://mirror.centos.org/centos~$NODEPOOL_CENTOS_MIRROR~" $REPO_PREFIX/delorean-deps.repo
  291. sudo sed -i -e "s~https://buildlogs.centos.org~$NODEPOOL_BUILDLOGS_CENTOS_PROXY~" $REPO_PREFIX/delorean-deps.repo
  292. sudo sed -i -e "s~https://trunk.rdoproject.org~$NODEPOOL_RDO_PROXY~" $REPO_PREFIX/delorean-deps.repo
  293. cat $REPO_PREFIX/delorean-deps.repo
  294. # Enable last known good RDO Trunk Delorean repository
  295. sudo curl -fLvo $REPO_PREFIX/delorean.repo $DELOREAN_REPO_URL/$DELOREAN_REPO_FILE
  296. sudo sed -i -e 's%priority=.*%priority=20%' $REPO_PREFIX/delorean.repo
  297. sudo sed -i -e "s~https://trunk.rdoproject.org~$NODEPOOL_RDO_PROXY~" $REPO_PREFIX/delorean.repo
  298. cat $REPO_PREFIX/delorean.repo
  299. # Enable latest RDO Trunk Delorean repository if not promotion job
  300. if [[ $CACHEUPLOAD != 1 ]]; then
  301. sudo curl -fLvo $REPO_PREFIX/delorean-current.repo https://trunk.rdoproject.org/centos7/current/delorean.repo
  302. sudo sed -i -e 's%priority=.*%priority=10%' $REPO_PREFIX/delorean-current.repo
  303. sudo sed -i 's/\[delorean\]/\[delorean-current\]/' $REPO_PREFIX/delorean-current.repo
  304. sudo sed -i -e "s~https://trunk.rdoproject.org~$NODEPOOL_RDO_PROXY~" $REPO_PREFIX/delorean-current.repo
  305. sudo /bin/bash -c "cat <<-EOF>>$REPO_PREFIX/delorean-current.repo
  306. includepkgs=diskimage-builder,instack,instack-undercloud,os-apply-config,os-collect-config,os-net-config,os-refresh-config,python-tripleoclient,openstack-tripleo-*,openstack-puppet-modules,puppet-*
  307. EOF"
  308. else
  309. # Create empty delorean-current for dib image building
  310. sudo sh -c "> $REPO_PREFIX/delorean-current.repo"
  311. fi
  312. cat $REPO_PREFIX/delorean-current.repo
  313. else
  314. # Enable the Delorean Deps repository
  315. sudo curl -fLvo $REPO_PREFIX/delorean-deps.repo https://trunk.rdoproject.org/centos7-$STABLE_RELEASE/delorean-deps.repo
  316. sudo sed -i -e 's%priority=.*%priority=30%' $REPO_PREFIX/delorean-deps.repo
  317. sudo sed -i -e "s~http://mirror.centos.org/centos~$NODEPOOL_CENTOS_MIRROR~" $REPO_PREFIX/delorean-deps.repo
  318. sudo sed -i -e "s~https://buildlogs.centos.org~$NODEPOOL_BUILDLOGS_CENTOS_PROXY~" $REPO_PREFIX/delorean-deps.repo
  319. cat $REPO_PREFIX/delorean-deps.repo
  320. # Enable delorean current for the stable version
  321. sudo curl -fLvo $REPO_PREFIX/delorean.repo $DELOREAN_STABLE_REPO_URL/$DELOREAN_REPO_FILE
  322. sudo sed -i -e 's%priority=.*%priority=20%' $REPO_PREFIX/delorean.repo
  323. sudo sed -i -e "s~https://trunk.rdoproject.org~$NODEPOOL_RDO_PROXY~" $REPO_PREFIX/delorean.repo
  324. cat $REPO_PREFIX/delorean.repo
  325. # Create empty delorean-current for dib image building
  326. sudo sh -c "> $REPO_PREFIX/delorean-current.repo"
  327. cat $REPO_PREFIX/delorean-current.repo
  328. fi
  329. # Install the yum-plugin-priorities package so that the Delorean repository
  330. # takes precedence over the main RDO repositories.
  331. sudo yum -y install yum-plugin-priorities
  332. # Make sure EPEL is uninstalled.
  333. if rpm --quiet -q epel-release; then
  334. sudo rpm -e epel-release
  335. fi
  336. sudo yum clean all
  337. sudo yum makecache
  338. log "Repository setup - DONE."
  339. }
  340. function delorean_setup {
  341. log "Delorean setup"
  342. # Install delorean as per combination of toci-instack and delorean docs
  343. sudo yum install -y createrepo git mock rpm-build yum-plugin-priorities yum-utils gcc rpmdevtools redhat-rpm-config
  344. # NOTE(pabelanger): Check if virtualenv is already install, if not install
  345. # from packages.
  346. if ! command -v virtualenv ; then
  347. sudo yum install -y python-virtualenv
  348. fi
  349. # Workaround until https://review.opendev.org/#/c/311734/ is merged and a new image is built
  350. sudo yum install -y libffi-devel openssl-devel
  351. # Add the current user to the mock group
  352. sudo usermod -G mock -a $(id -nu)
  353. mkdir -p $TRIPLEO_ROOT
  354. [ -d $TRIPLEO_ROOT/delorean ] || git clone https://github.com/softwarefactory-project/DLRN.git $TRIPLEO_ROOT/delorean
  355. pushd $TRIPLEO_ROOT/delorean
  356. sudo rm -rf data commits.sqlite
  357. mkdir -p data
  358. sed -i -e "s%reponame=.*%reponame=delorean-ci%" projects.ini
  359. sed -i -e "s%target=.*%target=centos%" projects.ini
  360. # Remove the rpm install test to speed up delorean (our ci test will to this)
  361. if [ -f scripts/build_rpm.sh ]; then
  362. # DLRN < 0.8.0
  363. sed -i -e 's%--postinstall%%' scripts/build_rpm.sh
  364. else
  365. # This is an option in DLRN since 0.8.0 for the mock build driver
  366. sed -i -e 's/^#install_after_build=1.*/install_after_build=0/' projects.ini
  367. fi
  368. virtualenv venv
  369. # NOTE(pabelanger): We need to update setuptools to the latest version for
  370. # CentOS 7. Also, pytz is not declared as a dependency so we need to
  371. # manually add it. Lastly, use pip install . to use wheel AFS pypi mirrors.
  372. ./venv/bin/pip install -U pip
  373. ./venv/bin/pip install -U setuptools
  374. ./venv/bin/pip install pytz
  375. ./venv/bin/pip install .
  376. popd
  377. log "Delorean setup - DONE."
  378. }
  379. function delorean_build {
  380. log "Delorean build"
  381. export PATH=/sbin:/usr/sbin:$PATH
  382. source $(dirname ${BASH_SOURCE[0]:-$0})/common_functions.sh
  383. pushd $TRIPLEO_ROOT/delorean
  384. if [ -n "$REVIEW_RELEASE" ]; then
  385. log "Building for release $REVIEW_RELEASE"
  386. # first check if we have a stable release
  387. sed -i -e "s%baseurl=.*%baseurl=$NODEPOOL_RDO_PROXY/centos7-$REVIEW_RELEASE%" projects.ini
  388. # RDO changed the distgit branch for stable releases starting from newton.
  389. sed -i -e "s%distro=.*%distro=$REVIEW_RELEASE-rdo%" projects.ini
  390. sed -i -e "s%source=.*%source=stable/$REVIEW_RELEASE%" projects.ini
  391. elif [ -n "$FEATURE_BRANCH" ]; then
  392. # next, check if we are testing for a feature branch
  393. log "Building for feature branch $FEATURE_BRANCH"
  394. sed -i -e "s%baseurl=.*%baseurl=$NODEPOOL_RDO_PROXY/centos7%" projects.ini
  395. sed -i -e "s%distro=.*%distro=rpm-$FEATURE_BRANCH%" projects.ini
  396. sed -i -e "s%source=.*%source=$FEATURE_BRANCH%" projects.ini
  397. else
  398. log "Building for master"
  399. sed -i -e "s%baseurl=.*%baseurl=$NODEPOOL_RDO_PROXY/centos7%" projects.ini
  400. sed -i -e "s%distro=.*%distro=rpm-master%" projects.ini
  401. sed -i -e "s%source=.*%source=master%" projects.ini
  402. fi
  403. sudo rm -rf data commits.sqlite
  404. mkdir -p data
  405. # build packages
  406. # loop through each of the projects listed in DELOREAN_BUILD_REFS, if it is a project we
  407. # are capable of building an rpm for then build it.
  408. # e.g. DELOREAN_BUILD_REFS="openstack/cinder openstack/heat etc.."
  409. for PROJ in $DELOREAN_BUILD_REFS ; do
  410. log "Building $PROJ"
  411. PROJ=$(filterref $PROJ)
  412. # Clone the repo if it doesn't yet exist
  413. if [ ! -d $TRIPLEO_ROOT/$PROJ ]; then
  414. git clone https://opendev.org/openstack/$PROJ.git $TRIPLEO_ROOT/$PROJ
  415. if [ ! -z "$REVIEW_RELEASE" ]; then
  416. pushd $TRIPLEO_ROOT/$PROJ
  417. git checkout -b stable/$REVIEW_RELEASE origin/stable/$REVIEW_RELEASE
  418. popd
  419. fi
  420. fi
  421. # Work around inconsistency where map-project-name expects oslo-*
  422. MAPPED_NAME=$(echo $PROJ | sed "s/oslo./oslo-/")
  423. MAPPED_PROJ=$(rdopkg findpkg $MAPPED_NAME | grep ^name | awk '{print $2}' || true)
  424. [ -e data/$MAPPED_PROJ ] && continue
  425. cp -r $TRIPLEO_ROOT/$PROJ data/$MAPPED_PROJ
  426. pushd data/$MAPPED_PROJ
  427. GITHASH=$(git rev-parse HEAD)
  428. # Set the branches delorean reads to the same git hash as PROJ has left for us
  429. for BRANCH in master origin/master stable/newton origin/stable/newton stable/ocata origin/stable/ocata stable/pike origin/stable/pike stable/queens origin/stable/queens; do
  430. git checkout -b $BRANCH || git checkout $BRANCH
  431. git reset --hard $GITHASH
  432. done
  433. popd
  434. set +e
  435. while true; do
  436. DELOREANCMD="./venv/bin/dlrn --config-file projects.ini --head-only --package-name $MAPPED_PROJ --local --use-public --build-env http_proxy=${http_proxy:-}"
  437. # Using sudo to su a command as ourselves to run the command with a new login
  438. # to ensure the addition to the mock group has taken effect.
  439. sudo su $(id -nu) -c "$DELOREANCMD"
  440. EXITCODE=$?
  441. # delorean exits with 2 if the error is a network glitch, we can retry
  442. if [ "$EXITCODE" == "2" ] ; then
  443. continue
  444. elif [ "$EXITCODE" == "0" ] ; then
  445. break
  446. fi
  447. set -e
  448. exit 1
  449. done
  450. set -e
  451. done
  452. popd
  453. log "Delorean build - DONE."
  454. }
  455. function undercloud {
  456. log "Undercloud install"
  457. sudo yum install -y python-tripleoclient ceph-ansible
  458. if [ ! -f ~/undercloud.conf ]; then
  459. cp -b -f $UNDERCLOUD_CONF ~/undercloud.conf
  460. else
  461. log "~/undercloud.conf already exists, not overwriting"
  462. fi
  463. # Hostname check, add to /etc/hosts if needed
  464. if ! grep -E "^127.0.0.1\s*$(hostname -f)" /etc/hosts; then
  465. sudo sed -i "s/127.0.0.1\s*\(.*\)/127.0.0.1\t$(hostname -f) $(hostname -s) \1/" /etc/hosts
  466. fi
  467. if ! grep -E "^::1\s*$(hostname -f)" /etc/hosts; then
  468. sudo sed -i "s/::1\s*\(.*\)/::1\t$(hostname -f) $(hostname -s) \1/" /etc/hosts
  469. fi
  470. openstack undercloud install
  471. # Masquerade traffic to external networks from controllers on baremetal undercloud
  472. # In ovb deployments, baremental nodes use undercloud as default route to reach DNS etc...
  473. if [ $OVB -eq 1 ]; then
  474. sudo iptables -A BOOTSTACK_MASQ -s 10.0.0.0/24 ! -d 10.0.0.0/24 -j MASQUERADE -t nat
  475. sudo iptables-save | sudo tee /etc/sysconfig/iptables
  476. fi
  477. log "Undercloud install - DONE."
  478. }
  479. function overcloud_images {
  480. log "Overcloud images"
  481. # This hack is no longer needed in ocata.
  482. if [[ "${STABLE_RELEASE}" =~ ^(newton)$ ]]; then
  483. # Ensure yum-plugin-priorities is installed
  484. # get the right path for diskimage-builder version
  485. COMMON_ELEMENTS_PATH=$(python -c '
  486. try:
  487. import diskimage_builder.paths
  488. diskimage_builder.paths.show_path("elements")
  489. except:
  490. print("/usr/share/diskimage-builder/elements")
  491. ')
  492. echo -e '#!/bin/bash\nyum install -y yum-plugin-priorities' | sudo tee ${COMMON_ELEMENTS_PATH}/yum/pre-install.d/99-tmphacks
  493. sudo chmod +x ${COMMON_ELEMENTS_PATH}/yum/pre-install.d/99-tmphacks
  494. fi
  495. # To install the undercloud instack-undercloud is run as root,
  496. # as a result all of the git repositories get cached to
  497. # ~root/.cache/image-create/source-repositories, lets not clone them again
  498. if [ -d ~root/.cache/image-create/source-repositories ] && \
  499. [ ! -d ~/.cache/image-create/source-repositories ] ; then
  500. sudo cp -r ~root/.cache/image-create/source-repositories ~/.cache/image-create/source-repositories
  501. sudo chown -R $(id -u) ~/.cache/image-create/source-repositories
  502. fi
  503. export DIB_YUM_REPO_CONF=$OVERCLOUD_IMAGES_DIB_YUM_REPO_CONF
  504. log "Overcloud images saved in $OVERCLOUD_IMAGES_PATH"
  505. pushd $OVERCLOUD_IMAGES_PATH
  506. log "OVERCLOUD_IMAGES_DIB_YUM_REPO_CONF=$OVERCLOUD_IMAGES_DIB_YUM_REPO_CONF"
  507. if [[ "${STABLE_RELEASE}" =~ ^(newton)$ ]] ; then
  508. OVERCLOUD_IMAGES_ARGS="$OVERCLOUD_IMAGES_LEGACY_ARGS"
  509. fi
  510. DIB_YUM_REPO_CONF=$OVERCLOUD_IMAGES_DIB_YUM_REPO_CONF \
  511. openstack overcloud image build $OVERCLOUD_IMAGES_ARGS 2>&1 | \
  512. tee -a overcloud-image-build.log
  513. stackrc_check
  514. openstack overcloud image upload --update-existing
  515. popd
  516. log "Overcloud images - DONE."
  517. }
  518. function register_nodes {
  519. log "Register nodes"
  520. if [ ! -f $INSTACKENV_JSON_PATH ]; then
  521. echo Could not find instackenv.json at $INSTACKENV_JSON_PATH
  522. echo Specify the path to instackenv.json with '$INSTACKENV_JSON_PATH'
  523. exit 1
  524. fi
  525. stackrc_check
  526. if [ "$INTROSPECT_NODES" = 1 ]; then
  527. # Keep the nodes in manageable state so that they may be
  528. # introspected later.
  529. openstack overcloud node import $INSTACKENV_JSON_PATH
  530. else
  531. openstack overcloud node import $INSTACKENV_JSON_PATH --provide
  532. fi
  533. ironic node-list
  534. log "Register nodes - DONE."
  535. }
  536. function introspect_nodes {
  537. log "Introspect nodes"
  538. stackrc_check
  539. # Note: Unlike the legacy bulk command, overcloud node
  540. # introspect will only run on nodes in the 'manageable'
  541. # provisioning state.
  542. openstack overcloud node introspect --all-manageable
  543. openstack overcloud node provide --all-manageable
  544. log "Introspect nodes - DONE."
  545. }
  546. function overcloud_deploy {
  547. log "Overcloud deploy"
  548. # Force use of --templates
  549. if [[ ! $OVERCLOUD_DEPLOY_ARGS =~ --templates ]]; then
  550. OVERCLOUD_DEPLOY_ARGS="$OVERCLOUD_DEPLOY_ARGS --templates"
  551. fi
  552. stackrc_check
  553. OVERCLOUD_DEPLOY_ARGS="$OVERCLOUD_DEPLOY_ARGS $OVERCLOUD_VALIDATE_ARGS"
  554. # Set dns server for the overcloud nodes
  555. subnet_id=$(openstack network list -f value -c Name -c Subnets | grep ctlplane | cut -d " " -f 2)
  556. neutron subnet-update $subnet_id --dns-nameserver $(cat /etc/resolv.conf | grep nameserver | awk '{ print $2 }' | sed ':a;N;$!ba;s/\n/ --dns-nameserver /g')
  557. if [[ $USE_CONTAINERS == 1 ]]; then
  558. if ! glance image-list | grep -q atomic-image; then
  559. wget --progress=dot:mega $ATOMIC_URL
  560. glance image-create --name atomic-image --file `basename $ATOMIC_URL` --disk-format qcow2 --container-format bare
  561. fi
  562. #TODO: When container job is changed to network-isolation remove this
  563. neutron subnet-update $(neutron net-list | grep ctlplane | cut -d ' ' -f 6) --dns-nameserver 8.8.8.8
  564. OVERCLOUD_DEPLOY_ARGS="$OVERCLOUD_DEPLOY_ARGS $CONTAINER_ARGS"
  565. fi
  566. log "unsetting any http proxy"
  567. unset http_proxy
  568. log "Overcloud create started."
  569. exitval=0
  570. log "Deploy command arguments: $OVERCLOUD_DEPLOY_ARGS"
  571. openstack overcloud deploy $OVERCLOUD_DEPLOY_ARGS || exitval=1
  572. if [ $exitval -eq 1 ]; then
  573. log "Overcloud create - FAILED!"
  574. exit 1
  575. fi
  576. log "Overcloud create - DONE."
  577. }
  578. function undercloud_upgrade {
  579. log "Undercloud upgrade"
  580. # Setup repositories
  581. repo_setup
  582. # In pike and above this is handled by the pre-upgrade hook
  583. if [[ "$STABLE_RELEASE" =~ ^(newton|ocata)$ ]]; then
  584. sudo systemctl stop openstack-*
  585. sudo systemctl stop neutron-*
  586. sudo systemctl stop openvswitch
  587. sudo systemctl stop httpd
  588. fi
  589. # tripleo cli needs to be updated first
  590. sudo yum -y update python-tripleoclient
  591. # Upgrade the undercloud
  592. openstack undercloud upgrade
  593. log "Undercloud upgrade - Done."
  594. }
  595. function overcloud_update {
  596. # Force use of --templates
  597. if [[ ! $OVERCLOUD_UPDATE_ARGS =~ --templates ]]; then
  598. OVERCLOUD_UPDATE_ARGS="$OVERCLOUD_UPDATE_ARGS --templates"
  599. fi
  600. stackrc_check
  601. if openstack stack show "$OVERCLOUD_NAME" | grep "stack_status " | egrep "(CREATE|UPDATE)_COMPLETE"; then
  602. FILE_PREFIX=$(date "+overcloud-update-resources-%s")
  603. BEFORE_FILE="/tmp/${FILE_PREFIX}-before.txt"
  604. AFTER_FILE="/tmp/${FILE_PREFIX}-after.txt"
  605. # This is an update, so if enabled, compare the before/after resource lists
  606. if [ ! -z "$OVERCLOUD_UPDATE_CHECK" ]; then
  607. openstack stack resource list -n5 overcloud | awk '{print $2, $4, $6}' | sort > $BEFORE_FILE
  608. fi
  609. log "Overcloud update started."
  610. exitval=0
  611. openstack overcloud deploy $OVERCLOUD_UPDATE_ARGS || exitval=1
  612. if [ $exitval -eq 1 ]; then
  613. log "Overcloud update - FAILED!"
  614. exit 1
  615. fi
  616. log "Overcloud update - DONE."
  617. if [ ! -z "$OVERCLOUD_UPDATE_CHECK" ]; then
  618. openstack stack resource list -n5 overcloud | awk '{print $2, $4, $6}' | sort > $AFTER_FILE
  619. diff_rsrc=$(diff $BEFORE_FILE $AFTER_FILE)
  620. if [ ! -z "$diff_rsrc" ]; then
  621. log "Overcloud update - Completed but unexpected resource differences: $diff_rsrc"
  622. exit 1
  623. fi
  624. fi
  625. log "Overcloud update - DONE."
  626. if [[ $OVERCLOUD_UPDATE_RM_FILES == 1 ]]; then
  627. rm -f $BEFORE_FILE $AFTER_FILE
  628. fi
  629. else
  630. log "Overcloud FAILED - No stack $OVERCLOUD_NAME."
  631. exit 1
  632. fi
  633. }
  634. function overcloud_upgrade {
  635. stackrc_check
  636. if openstack stack show "$OVERCLOUD_NAME" 2>&1 > /dev/null ; then
  637. log "Create overcloud repo template file"
  638. /bin/bash -c "cat <<EOF>$HOME/init-repo.yaml
  639. parameter_defaults:
  640. UpgradeInitCommand: |
  641. set -e
  642. # For some reason '$HOME' is not defined when the Heat agent executes this
  643. # script and tripleo.sh expects it. Just reuse the same value from the
  644. # current undercloud user.
  645. yum clean all
  646. export STABLE_RELEASE=$STABLE_RELEASE
  647. HOME=$HOME $TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --repo-setup
  648. yum clean all
  649. yum install -y python-heat-agent-*
  650. # TODO: (slagle)
  651. # remove the --noscripts install of openstack-tripleo-image-elements
  652. # once https://review.rdoproject.org/r/4225 merges
  653. pushd /tmp
  654. yumdownloader openstack-tripleo-image-elements
  655. rpm -Uvh --noscripts --force ./openstack-tripleo-image-elements*
  656. rm -f openstack-tripleo-image-elements*
  657. popd
  658. # FIXME: matbu
  659. # Remove those packages is temporary workaround since the fix in
  660. # https://bugs.launchpad.net/tripleo/+bug/1649284
  661. # will be release and landed in the packages
  662. yum remove -y python-UcsSdk openstack-neutron-bigswitch-agent python-networking-bigswitch openstack-neutron-bigswitch-lldp python-networking-odl
  663. EOF"
  664. log "Overcloud upgrade started."
  665. log "Upgrade command arguments: $OVERCLOUD_UPGRADE_ARGS"
  666. log "Execute major upgrade."
  667. openstack overcloud deploy $OVERCLOUD_UPGRADE_ARGS
  668. log "Major upgrade - DONE."
  669. if openstack stack show "$OVERCLOUD_NAME" | grep "stack_status " | egrep "UPDATE_COMPLETE"; then
  670. log "Major Upgrade - DONE."
  671. else
  672. log "Major Upgrade FAILED."
  673. exit 1
  674. fi
  675. else
  676. log "Overcloud upgrade FAILED - No stack $OVERCLOUD_NAME."
  677. exit 1
  678. fi
  679. }
  680. function overcloud_upgrade_converge {
  681. stackrc_check
  682. if openstack stack show "$OVERCLOUD_NAME" 2>&1 > /dev/null; then
  683. log "Overcloud upgrade converge started."
  684. log "Upgrade command arguments: $OVERCLOUD_UPGRADE_CONVERGE_ARGS"
  685. log "Execute major upgrade converge."
  686. openstack overcloud deploy $OVERCLOUD_UPGRADE_CONVERGE_ARGS
  687. log "Major upgrade converge - DONE."
  688. if openstack stack show "$OVERCLOUD_NAME" | grep "stack_status " | egrep "UPDATE_COMPLETE"; then
  689. log "Major Upgrade converge - DONE."
  690. else
  691. log "Major Upgrade converge FAILED."
  692. exit 1
  693. fi
  694. else
  695. log "Overcloud upgrade converge FAILED - No stack $OVERCLOUD_NAME."
  696. exit 1
  697. fi
  698. }
  699. function overcloud_delete {
  700. log "Overcloud delete"
  701. stackrc_check
  702. OVERCLOUD_ID=$(openstack stack list | grep "$OVERCLOUD_NAME" | awk '{print $2}')
  703. wait_command="openstack stack show $OVERCLOUD_ID"
  704. openstack stack delete --yes "$OVERCLOUD_NAME"
  705. if $($(dirname $0)/wait_for -w $OVERCLOUD_DELETE_TIMEOUT -d 10 -s "DELETE_COMPLETE" -- "$wait_command"); then
  706. log "Overcloud $OVERCLOUD_ID DELETE_COMPLETE"
  707. else
  708. log "Overcloud $OVERCLOUD_ID delete failed or timed out:"
  709. openstack stack show $OVERCLOUD_ID
  710. exit 1
  711. fi
  712. if [[ "${STABLE_RELEASE}" != "mitaka" ]] ; then
  713. openstack overcloud plan delete "$OVERCLOUD_NAME" && exitval=0 || exitval=1
  714. if [ ${exitval} -eq 0 ]; then
  715. log "Overcloud $OVERCLOUD_ID plan delete SUCCESS"
  716. else
  717. log "Overcloud $OVERCLOUD_ID plan delete FAILED"
  718. exit 1
  719. fi
  720. fi
  721. }
  722. function run_cmd {
  723. if ! $@; then
  724. echo "Command: $@ FAILED" >&2
  725. exit 1
  726. else
  727. echo "Command: $@ OK"
  728. fi
  729. }
  730. function overcloud_sanitytest_create {
  731. ENABLED_SERVICES=$@
  732. for service in $ENABLED_SERVICES; do
  733. case $service in
  734. "keystone" )
  735. run_cmd openstack user create ${SANITYTEST_CONTENT_NAME}
  736. run_cmd openstack user list
  737. ;;
  738. "glance_api" )
  739. run_cmd openstack image create ${SANITYTEST_CONTENT_NAME}
  740. run_cmd openstack image list
  741. ;;
  742. "neutron_api" )
  743. run_cmd openstack network create ${SANITYTEST_CONTENT_NAME}
  744. run_cmd openstack network list
  745. ;;
  746. "cinder_api" )
  747. run_cmd openstack volume create ${SANITYTEST_CONTENT_NAME} --size 1
  748. run_cmd openstack volume list
  749. ;;
  750. "heat_api" )
  751. echo "heat_template_version: newton" > /tmp/${SANITYTEST_CONTENT_NAME}.yaml
  752. openstack stack create ${SANITYTEST_CONTENT_NAME} --template /tmp/${SANITYTEST_CONTENT_NAME}.yaml
  753. openstack stack list
  754. ;;
  755. "swift_proxy" )
  756. openstack container create ${SANITYTEST_CONTENT_NAME}
  757. openstack container list
  758. ;;
  759. "sahara_api" )
  760. # glance_api must also be enabled
  761. run_cmd openstack image create sahara_${SANITYTEST_CONTENT_NAME}
  762. run_cmd openstack dataprocessing image register sahara_${SANITYTEST_CONTENT_NAME} --username centos
  763. run_cmd openstack dataprocessing image list
  764. ;;
  765. esac
  766. done
  767. }
  768. function overcloud_sanitytest_check {
  769. ENABLED_SERVICES=$@
  770. for service in $ENABLED_SERVICES; do
  771. case $service in
  772. "keystone" )
  773. run_cmd openstack user show ${SANITYTEST_CONTENT_NAME}
  774. ;;
  775. "glance_api" )
  776. run_cmd openstack image show ${SANITYTEST_CONTENT_NAME}
  777. ;;
  778. "neutron_api" )
  779. run_cmd openstack network show ${SANITYTEST_CONTENT_NAME}
  780. ;;
  781. "cinder_api" )
  782. run_cmd openstack volume show ${SANITYTEST_CONTENT_NAME}
  783. ;;
  784. "heat_api" )
  785. run_cmd openstack stack show ${SANITYTEST_CONTENT_NAME}
  786. # FIXME(shardy): It'd be good to add pre/post upgrade checks
  787. # on the actual version, but this is still good for debugging
  788. run_cmd openstack orchestration template version list
  789. ;;
  790. "swift_proxy" )
  791. run_cmd openstack container show ${SANITYTEST_CONTENT_NAME}
  792. ;;
  793. "sahara_api" )
  794. run_cmd openstack dataprocessing image show sahara_${SANITYTEST_CONTENT_NAME}
  795. ;;
  796. esac
  797. done
  798. }
  799. function overcloud_sanitytest_cleanup {
  800. ENABLED_SERVICES=$@
  801. for service in $ENABLED_SERVICES; do
  802. case $service in
  803. "keystone" )
  804. echo "Sanity test keystone"
  805. run_cmd openstack user delete ${SANITYTEST_CONTENT_NAME}
  806. ;;
  807. "glance_api" )
  808. run_cmd openstack image delete ${SANITYTEST_CONTENT_NAME}
  809. ;;
  810. "neutron_api" )
  811. run_cmd openstack network delete ${SANITYTEST_CONTENT_NAME}
  812. ;;
  813. "cinder_api" )
  814. run_cmd openstack volume delete ${SANITYTEST_CONTENT_NAME}
  815. ;;
  816. "heat_api" )
  817. run_cmd openstack stack delete --yes ${SANITYTEST_CONTENT_NAME}
  818. ;;
  819. "swift_proxy" )
  820. run_cmd openstack container delete ${SANITYTEST_CONTENT_NAME}
  821. ;;
  822. "sahara_api" )
  823. run_cmd openstack dataprocessing image unregister sahara_${SANITYTEST_CONTENT_NAME}
  824. run_cmd openstack image delete sahara_${SANITYTEST_CONTENT_NAME}
  825. ;;
  826. esac
  827. done
  828. }
  829. function overcloud_sanitytest {
  830. log "Overcloud sanitytest"
  831. exitval=0
  832. stackrc_check
  833. if openstack stack show "$OVERCLOUD_NAME" | grep "stack_status " | egrep -q "(CREATE|UPDATE)_COMPLETE"; then
  834. ENABLED_SERVICES=$(openstack stack output show overcloud EnabledServices -f json | \
  835. jq -r ".output_value" | jq '.Controller | .[]' | tr "\n" " " | sed "s/\"//g")
  836. echo "Sanity Test, ENABLED_SERVICES=$ENABLED_SERVICES"
  837. overcloudrc_check
  838. if [ "$OVERCLOUD_SANITYTEST_SKIP_CREATE" != 1 ]; then
  839. overcloud_sanitytest_create $ENABLED_SERVICES
  840. fi
  841. overcloud_sanitytest_check $ENABLED_SERVICES
  842. if [ "$OVERCLOUD_SANITYTEST_SKIP_CLEANUP" != 1 ]; then
  843. overcloud_sanitytest_cleanup $ENABLED_SERVICES
  844. fi
  845. if [ $exitval -eq 0 ]; then
  846. log "Overcloud sanitytest SUCCEEDED"
  847. else
  848. log "Overcloud sanitytest FAILED"
  849. fi
  850. exit $exitval
  851. else
  852. log "Overcloud sanitytest FAILED - No stack $OVERCLOUD_NAME."
  853. exit 1
  854. fi
  855. }
  856. function cleanup_pingtest {
  857. log "Overcloud pingtest; cleaning environment"
  858. overcloudrc_check
  859. wait_command="openstack stack show tenant-stack"
  860. openstack stack delete --yes tenant-stack || true
  861. if $(dirname $0)/wait_for -w 300 -d 10 -s "Stack not found" -- "$wait_command"; then
  862. log "Overcloud pingtest - deleted the tenant-stack heat stack"
  863. else
  864. log "Overcloud pingtest - time out waiting to delete tenant heat stack, please check manually"
  865. fi
  866. log "Overcloud pingtest - cleaning all 'pingtest_*' images"
  867. openstack image list | grep pingtest | awk '{print $2}' | xargs -r -n1 openstack image delete || true
  868. log "Overcloud pingtest - cleaning demo network 'nova'"
  869. neutron net-delete nova || true
  870. }
  871. function overcloud_pingtest {
  872. log "Overcloud pingtest"
  873. exitval=0
  874. stackrc_check
  875. SUBNET_ID=$(openstack network list -c Subnets -c Name -f value | grep ctlplane | awk {'print $2'})
  876. CTLPLANE_CIDR=$(neutron subnet-show $SUBNET_ID -F cidr -f value)
  877. CTLPLANE_NET=$(echo $CTLPLANE_CIDR | awk -F "." {'print $1"."$2"."$3'})
  878. overcloudrc_check
  879. if [ "$SKIP_PINGTEST_CLEANUP" != 1 ]; then
  880. cleanup_pingtest
  881. else
  882. log "Skipping pre-pingtest cleanup because --skip-pingtest-cleanup was specified."
  883. fi
  884. # NOTE(bnemec): We have to use the split cirros image here to avoid
  885. # https://bugs.launchpad.net/cirros/+bug/1312199 With the separate
  886. # kernel and ramdisk Nova will add the necessary kernel param for us.
  887. IMAGE_PATH=$OVERCLOUD_IMAGES_PATH/cirros.img
  888. INITRAMFS_PATH=$OVERCLOUD_IMAGES_PATH/cirros.initramfs
  889. KERNEL_PATH=$OVERCLOUD_IMAGES_PATH/cirros.kernel
  890. if [ ! -e $IMAGE_PATH -o ! -e $INITRAMFS_PATH -o ! -e $KERNEL_PATH ]; then
  891. log "Overcloud pingtest, trying to download Cirros image"
  892. curl -fLo $IMAGE_PATH http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-disk.img
  893. curl -fLo $INITRAMFS_PATH http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-initramfs
  894. curl -fLo $KERNEL_PATH http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-kernel
  895. fi
  896. log "Overcloud pingtest, uploading demo tenant image to glance"
  897. ramdisk_id=$(openstack image create pingtest_initramfs --public --container-format ari --disk-format ari --file $INITRAMFS_PATH | grep ' id ' | awk '{print $4}')
  898. kernel_id=$(openstack image create pingtest_kernel --public --container-format aki --disk-format aki --file $KERNEL_PATH | grep ' id ' | awk '{print $4}')
  899. openstack image create pingtest_image --public --container-format ami --disk-format ami --property kernel_id=$kernel_id --property ramdisk_id=$ramdisk_id --file $IMAGE_PATH
  900. log "Overcloud pingtest, creating external network"
  901. neutron net-create nova --shared --router:external=True --provider:network_type flat \
  902. --provider:physical_network datacentre
  903. FLOATING_IP_CIDR=${FLOATING_IP_CIDR:-$CTLPLANE_CIDR}
  904. FLOATING_IP_START=${FLOATING_IP_START:-"${CTLPLANE_NET}.50"}
  905. FLOATING_IP_END=${FLOATING_IP_END:-"${CTLPLANE_NET}.64"}
  906. EXTERNAL_NETWORK_GATEWAY=${EXTERNAL_NETWORK_GATEWAY:-"${CTLPLANE_NET}.1"}
  907. TENANT_STACK_DEPLOY_ARGS=${TENANT_STACK_DEPLOY_ARGS:-""}
  908. neutron subnet-create --name ext-subnet --allocation-pool start=$FLOATING_IP_START,end=$FLOATING_IP_END --disable-dhcp --gateway $EXTERNAL_NETWORK_GATEWAY nova $FLOATING_IP_CIDR
  909. # pingtest environment for scenarios jobs is in TripleO Heat Templates.
  910. if [ -e "${TRIPLEO_HEAT_TEMPLATES_ROOT}/ci/pingtests/${MULTINODE_ENV_NAME}.yaml" ]; then
  911. TENANT_PINGTEST_TEMPLATE="${TRIPLEO_HEAT_TEMPLATES_ROOT}/ci/pingtests/${MULTINODE_ENV_NAME}.yaml"
  912. elif [ -e "${TRIPLEO_HEAT_TEMPLATES_ROOT}/ci/pingtests/tenantvm_floatingip.yaml" ]; then
  913. TENANT_PINGTEST_TEMPLATE="${TRIPLEO_HEAT_TEMPLATES_ROOT}/ci/pingtests/tenantvm_floatingip.yaml"
  914. else
  915. # If the template is not found, we will get the template from the tripleo-ci location for backwards compatibility.
  916. TENANT_PINGTEST_TEMPLATE="${TRIPLEO_ROOT}/tripleo-ci/templates/tenantvm_floatingip.yaml"
  917. fi
  918. log "Overcloud pingtest, creating tenant-stack heat stack:"
  919. openstack stack create -f yaml -t $TENANT_PINGTEST_TEMPLATE $TENANT_STACK_DEPLOY_ARGS tenant-stack || exitval=1
  920. WAIT_FOR_COMMAND="openstack stack list | grep tenant-stack"
  921. # No point in waiting if the previous command failed.
  922. if [ ${exitval} -eq 0 ]; then
  923. # TODO(beagles): While the '-f' flag will short-circuit fail us, we'll
  924. # likely have to wait for service operations to timeout before the
  925. # stack gets marked as failed anyways. A CI oriented configuration for
  926. # some key services *might* work for 'fail faster', but where things
  927. # can be so slow already it might just cause more pain.
  928. #
  929. if $(dirname $0)/wait_for -w 300 -d 10 -s "CREATE_COMPLETE" -f "CREATE_FAILED" -- $WAIT_FOR_COMMAND; then
  930. log "Overcloud pingtest, heat stack CREATE_COMPLETE";
  931. vm1_ip=`openstack stack output show tenant-stack server1_public_ip | grep value | awk '{print $4}'`
  932. log "Overcloud pingtest, trying to ping the floating IPs $vm1_ip"
  933. if $(dirname $0)/wait_for -w 360 -d 10 -s "bytes from $vm1_ip" -- "ping -c 1 $vm1_ip" ; then
  934. ping -c 1 $vm1_ip
  935. log "Overcloud pingtest, SUCCESS"
  936. else
  937. ping -c 1 $vm1_ip || :
  938. nova show Server1 || :
  939. nova service-list || :
  940. neutron agent-list || :
  941. nova console-log Server1 || :
  942. log "Overcloud pingtest, FAIL"
  943. exitval=1
  944. fi
  945. else
  946. nova service-list || :
  947. neutron agent-list || :
  948. openstack stack show tenant-stack || :
  949. openstack stack event list -f table tenant-stack || :
  950. openstack stack resource list -n5 tenant-stack || :
  951. openstack stack failures list tenant-stack || :
  952. log "Overcloud pingtest, failed to create heat stack, trying cleanup"
  953. exitval=1
  954. fi
  955. else
  956. log "Overcloud pingtest, stack create command failed immediately"
  957. fi
  958. if [ "$SKIP_PINGTEST_CLEANUP" != 1 ]; then
  959. cleanup_pingtest
  960. else
  961. log "Overcloud pingtest, the resources created by the pingtest will remain until a new pingtest is executed."
  962. fi
  963. if [ $exitval -eq 0 ]; then
  964. log "Overcloud pingtest SUCCEEDED"
  965. else
  966. log "Overcloud pingtest FAILED"
  967. fi
  968. exit $exitval
  969. }
  970. function clean_tempest {
  971. neutron net-delete nova || echo "Cleaning tempest: no networks were created"
  972. }
  973. function tempest_run {
  974. log "Running tempest"
  975. stackrc_check
  976. CTLPLANE_CIDR=$(neutron net-list -c subnets -c name -f value | grep ctlplane | awk {'print $2'})
  977. CTLPLANE_NET=$(echo $CTLPLANE_CIDR | awk -F "." {'print $1"."$2"."$3'})
  978. overcloudrc_check
  979. clean_tempest
  980. root_dir=$(realpath $(dirname ${BASH_SOURCE[0]:-$0}))
  981. [[ ! -e $HOME/tempest ]] && git clone https://github.com/openstack/tempest $HOME/tempest
  982. pushd $HOME/tempest
  983. git checkout $TEMPEST_PINNED
  984. FLOATING_IP_CIDR=${FLOATING_IP_CIDR:-$CTLPLANE_CIDR}
  985. FLOATING_IP_START=${FLOATING_IP_START:-"${CTLPLANE_NET}.50"}
  986. FLOATING_IP_END=${FLOATING_IP_END:-"${CTLPLANE_NET}.64"}
  987. export EXTERNAL_NETWORK_GATEWAY=${EXTERNAL_NETWORK_GATEWAY:-"${CTLPLANE_NET}.1"}
  988. neutron net-create nova --shared --router:external=True --provider:network_type flat --provider:physical_network datacentre;
  989. neutron subnet-create --name ext-subnet --allocation-pool start=$FLOATING_IP_START,end=$FLOATING_IP_END --disable-dhcp --gateway $EXTERNAL_NETWORK_GATEWAY nova $FLOATING_IP_CIDR;
  990. sudo yum install -y libffi-devel openssl-devel python-virtualenv
  991. virtualenv --no-site-packages .venv
  992. $HOME/tempest/tools/with_venv.sh pip install -U pip setuptools
  993. $HOME/tempest/tools/with_venv.sh pip install junitxml httplib2 -r test-requirements.txt -r requirements.txt
  994. cp $root_dir/config_tempest.py $HOME/tempest/tools/
  995. cp $root_dir/api_discovery.py $HOME/tempest/tempest/common/
  996. cp $root_dir/default-overrides.conf $HOME/tempest/etc/
  997. sudo mkdir -p /var/log/tempest/ ||:
  998. sudo mkdir -p /etc/tempest/ ||:
  999. sudo chown $USER:$USER -R /var/log/tempest/
  1000. $HOME/tempest/tools/with_venv.sh python $HOME/tempest/tools/config_tempest.py \
  1001. --out etc/tempest.conf \
  1002. --debug \
  1003. --create \
  1004. --deployer-input ~/tempest-deployer-input.conf \
  1005. identity.uri $OS_AUTH_URL \
  1006. auth.use_dynamic_credentials true \
  1007. identity.admin_password $OS_PASSWORD \
  1008. compute.build_timeout 500 \
  1009. validation.image_ssh_user cirros \
  1010. orchestration.stack_owner_role _member_ \
  1011. network.build_timeout 500 \
  1012. volume.build_timeout 500 \
  1013. DEFAULT.log_file "/var/log/tempest/tempest.log" \
  1014. $TEMPEST_ADD_CONFIG
  1015. sudo cp $HOME/tempest/etc/tempest.conf /etc/tempest/tempest.conf
  1016. [[ ! -e $HOME/tempest/.testrepository ]] && $HOME/tempest/tools/with_venv.sh testr init
  1017. $HOME/tempest/tools/with_venv.sh testr run \
  1018. $TEMPEST_ARGS \
  1019. $TEMPEST_REGEX | \
  1020. tee >( $HOME/tempest/tools/with_venv.sh subunit2junitxml --output-to=/var/log/tempest/tempest.xml ) | \
  1021. $HOME/tempest/tools/with_venv.sh subunit-trace --no-failure-debug -f 2>&1 | \
  1022. tee /var/log/tempest/tempest_console.log && exitval=0 || exitval=$?
  1023. $HOME/tempest/tools/with_venv.sh subunit2html $HOME/tempest/.testrepository/$(ls -t $HOME/tempest/.testrepository/ | grep -e "[0-9]" | head -1) /var/log/tempest/tempest.html
  1024. exit ${exitval}
  1025. }
  1026. function clone {
  1027. local repo=$1
  1028. log "$0 requires $repo to be cloned at \$TRIPLEO_ROOT ($TRIPLEO_ROOT)"
  1029. mkdir -p $TRIPLEO_ROOT
  1030. if [ ! -d $TRIPLEO_ROOT/$(basename $repo) ]; then
  1031. echo "$repo not found at $TRIPLEO_ROOT/$repo, git cloning."
  1032. pushd $TRIPLEO_ROOT
  1033. git clone https://opendev.org/$repo
  1034. popd
  1035. else
  1036. echo "$repo found at $TRIPLEO_ROOT/$repo, nothing to do."
  1037. fi
  1038. }
  1039. # This function creates an internal gre bridge to connect all external
  1040. # network bridges across the compute and network nodes.
  1041. # bridge_name: Bridge name on each host for logical l2 network
  1042. # connectivity.
  1043. # host_ip: ip address of the bridge host which is reachable for all peer
  1044. # the hub for all of our spokes.
  1045. # set_ips: Whether or not to set l3 addresses on our logical l2 network.
  1046. # This can be helpful for setting up routing tables.
  1047. # offset: starting value for gre tunnel key and the ip addr suffix
  1048. # The next two parameters are only used if set_ips is "True".
  1049. # pub_addr_prefix: The IPv4 address three octet prefix used to give compute
  1050. # nodes non conflicting addresses on the pub_if_name'd
  1051. # network. Should be provided as X.Y.Z. Offset will be
  1052. # applied to this as well as the below mask to get the
  1053. # resulting address.
  1054. # pub_addr_mask: the CIDR mask less the '/' for the IPv4 addresses used
  1055. # above.
  1056. # every additional parameter is considered as a peer host (spokes)
  1057. #
  1058. # For OVS troubleshooting needs:
  1059. # http://www.yet.org/2014/09/openvswitch-troubleshooting/
  1060. #
  1061. function ovs_vxlan_bridge {
  1062. if is_suse; then
  1063. local ovs_package='openvswitch'
  1064. local ovs_service='openvswitch'
  1065. elif is_fedora; then
  1066. local ovs_package='openvswitch openstack-selinux'
  1067. local ovs_service='openvswitch'
  1068. elif uses_debs; then
  1069. local ovs_package='openvswitch-switch'
  1070. local ovs_service='openvswitch-switch'
  1071. else
  1072. echo "Unsupported platform, can't determine openvswitch service"
  1073. exit 1
  1074. fi
  1075. local install_ovs_deps="source $BASE/new/devstack/functions-common; \
  1076. install_package ${ovs_package}; \
  1077. restart_service ${ovs_service}"
  1078. local mtu=1450
  1079. local bridge_name=$1
  1080. local host_ip=$2
  1081. local set_ips=$3
  1082. local offset=$4
  1083. if [[ "$set_ips" == "True" ]] ; then
  1084. local pub_addr_prefix=$5
  1085. local pub_addr_mask=$6
  1086. shift 6
  1087. else
  1088. shift 4
  1089. fi
  1090. local peer_ips=$@
  1091. # neutron uses 1:1000 with default devstack configuration, avoid overlap
  1092. local additional_vni_offset=1000000
  1093. eval $install_ovs_deps
  1094. # create a bridge, just like you would with 'brctl addbr'
  1095. # if the bridge exists, --may-exist prevents ovs from returning an error
  1096. sudo ovs-vsctl --may-exist add-br $bridge_name
  1097. # as for the mtu, look for notes on lp#1301958 in devstack-vm-gate.sh
  1098. sudo ip link set mtu $mtu dev $bridge_name
  1099. if [[ "$set_ips" == "True" ]] ; then
  1100. echo "Set bridge: ${bridge_name}"
  1101. if ! sudo ip addr show dev ${bridge_name} | grep -q \
  1102. ${pub_addr_prefix}.${offset}/${pub_addr_mask} ; then
  1103. sudo ip addr add ${pub_addr_prefix}.${offset}/${pub_addr_mask} \
  1104. dev ${bridge_name}
  1105. fi
  1106. fi
  1107. sudo ip link set dev $bridge_name up
  1108. for node_ip in $peer_ips; do
  1109. offset=$(( offset+1 ))
  1110. vni=$(( offset + additional_vni_offset ))
  1111. # For reference on how to setup a tunnel using OVS see:
  1112. # http://openvswitch.org/support/config-cookbooks/port-tunneling/
  1113. # The command below is equivalent to the sequence of ip/brctl commands
  1114. # where an interface of vxlan type is created first, and then plugged into
  1115. # the bridge; options are command specific configuration key-value pairs.
  1116. #
  1117. # Create the vxlan tunnel for the Controller/Network Node:
  1118. # This establishes a tunnel between remote $node_ip to local $host_ip
  1119. # uniquely identified by a key $offset
  1120. sudo ovs-vsctl --may-exist add-port $bridge_name \
  1121. ${bridge_name}_${node_ip} \
  1122. -- set interface ${bridge_name}_${node_ip} type=vxlan \
  1123. options:remote_ip=${node_ip} \
  1124. options:key=${vni} \
  1125. options:local_ip=${host_ip}
  1126. # Now complete the vxlan tunnel setup for the Compute Node:
  1127. # Similarly this establishes the tunnel in the reverse direction
  1128. remote_command $node_ip "$install_ovs_deps"
  1129. remote_command $node_ip sudo ovs-vsctl --may-exist add-br $bridge_name
  1130. remote_command $node_ip sudo ip link set mtu $mtu dev $bridge_name
  1131. remote_command $node_ip sudo ovs-vsctl --may-exist add-port $bridge_name \
  1132. ${bridge_name}_${host_ip} \
  1133. -- set interface ${bridge_name}_${host_ip} type=vxlan \
  1134. options:remote_ip=${host_ip} \
  1135. options:key=${vni} \
  1136. options:local_ip=${node_ip}
  1137. if [[ "$set_ips" == "True" ]] ; then
  1138. if ! remote_command $node_ip sudo ip addr show dev ${bridge_name} | \
  1139. grep -q ${pub_addr_prefix}.${offset}/${pub_addr_mask} ; then
  1140. remote_command $node_ip sudo ip addr add \
  1141. ${pub_addr_prefix}.${offset}/${pub_addr_mask} \
  1142. dev ${bridge_name}
  1143. fi
  1144. fi
  1145. remote_command $node_ip sudo ip link set dev $bridge_name up
  1146. done
  1147. }
  1148. function multinode_setup {
  1149. log "Multinode Setup"
  1150. clone openstack-dev/devstack
  1151. clone openstack-infra/devstack-gate
  1152. # $BASE is expected by devstack/functions-common
  1153. # which is sourced by devstack-gate/functions.sh
  1154. # It should be the parent directory of the "new" directory where
  1155. # zuul-cloner has checked out the repositories
  1156. export BASE
  1157. export TRIPLEO_ROOT
  1158. log "Sourcing devstack-gate/functions.sh"
  1159. set +u
  1160. source $TRIPLEO_ROOT/devstack-gate/functions.sh
  1161. set -u
  1162. PUB_BRIDGE_NAME=${PUB_BRIDGE_NAME:-"br-ex"}
  1163. local primary_node
  1164. primary_node=$(cat /etc/nodepool/primary_node_private)
  1165. local sub_nodes
  1166. sub_nodes=$(cat /etc/nodepool/sub_nodes_private)
  1167. for ip in $sub_nodes; do
  1168. # Do repo setup so openvswitch package is available on subnodes. Will
  1169. # be installed by ovs_vxlan_bridge function below.
  1170. log "Running --repo-setup on $ip"
  1171. ssh $SSH_OPTIONS -t -i /etc/nodepool/id_rsa $ip \
  1172. TRIPLEO_ROOT=$TRIPLEO_ROOT \
  1173. $TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --repo-setup
  1174. done
  1175. # Create OVS vxlan bridges
  1176. # If br-ctlplane already exists on this node, we need to bring it down
  1177. # first, then bring it back up after calling ovs_vxlan_bridge. This ensures
  1178. # that the route added to br-ex by ovs_vxlan_bridge will be preferred over
  1179. # the br-ctlplane route. If it's not preferred, multinode connectivity
  1180. # across the vxlan bridge will not work.
  1181. if [ -f /etc/sysconfig/network-scripts/ifcfg-br-ctlplane ]; then
  1182. sudo ifdown br-ctlplane
  1183. fi
  1184. set +u
  1185. log "Running ovs_vxlan_bridge"
  1186. ovs_vxlan_bridge $PUB_BRIDGE_NAME $primary_node "True" 2 192.168.24 24 $sub_nodes
  1187. set -u
  1188. log "Setting $PUB_BRIDGE_NAME up on $primary_node"
  1189. sudo ip link set dev $PUB_BRIDGE_NAME up
  1190. sudo ip link set dev $PUB_BRIDGE_NAME mtu $MTU
  1191. if [ -f /etc/sysconfig/network-scripts/ifcfg-br-ctlplane ]; then
  1192. sudo ifup br-ctlplane
  1193. fi
  1194. # Restart neutron-openvswitch-agent if it's enabled, since it may have
  1195. # terminated when br-ctlplane was down
  1196. if [ "$(sudo systemctl is-enabled neutron-openvswitch-agent)" = 'enabled' ]; then
  1197. sudo systemctl reset-failed neutron-openvswitch-agent
  1198. sudo systemctl restart neutron-openvswitch-agent
  1199. fi
  1200. local ping_command="ping -c 6 -W 3 192.168.24.2"
  1201. for ip in $sub_nodes; do
  1202. log "Setting $PUB_BRIDGE_NAME up on $ip"
  1203. ssh $SSH_OPTIONS -t -i /etc/nodepool/id_rsa $ip \
  1204. sudo ip link set dev $PUB_BRIDGE_NAME up
  1205. ssh $SSH_OPTIONS -t -i /etc/nodepool/id_rsa $ip \
  1206. sudo ip link set dev $PUB_BRIDGE_NAME mtu $MTU
  1207. log "Pinging from $ip"
  1208. if ! remote_command $ip $ping_command; then
  1209. log "Pinging from $ip failed, restarting openvswitch"
  1210. remote_command $ip sudo systemctl restart openvswitch
  1211. if ! remote_command $ip $ping_command; then
  1212. log "Pinging from $ip still failed after restarting openvswitch"
  1213. exit 1
  1214. fi
  1215. fi
  1216. done
  1217. log "Multinode Setup - DONE".
  1218. }
  1219. function undercloud_sanity_check {
  1220. set -x
  1221. stackrc_check
  1222. openstack user list
  1223. openstack catalog list
  1224. nova service-list
  1225. glance image-list
  1226. neutron subnet-list
  1227. neutron net-list
  1228. neutron agent-list
  1229. if [[ ! "${STABLE_RELEASE}" =~ ^(newton|ocata) ]]; then
  1230. # This verifies that at least one conductor comes up, and at least one
  1231. # IPMI-based driver was successfully enabled.
  1232. ipmi_drivers="$(grep -c ipmi <(ironic driver-list))"
  1233. if [[ $ipmi_drivers -eq 0 ]]; then
  1234. log "ERROR: Check ironic driver-list"
  1235. exit 1
  1236. fi
  1237. else
  1238. ironic node-list
  1239. fi
  1240. openstack stack list
  1241. ui_sanity_check
  1242. set +x
  1243. }
  1244. function bootstrap_subnodes {
  1245. log "WARNING: Bootstrap subnodes is deprecated and will be removed. "
  1246. local sub_nodes
  1247. sub_nodes=$(cat /etc/nodepool/sub_nodes_private)
  1248. bootstrap_subnodes_repos
  1249. local bootstrap_script
  1250. if [ "$BOOTSTRAP_SUBNODES_MINIMAL" = "1" ]; then
  1251. bootstrap_script=bootstrap-overcloud-full-minimal.sh
  1252. else
  1253. bootstrap_script=bootstrap-overcloud-full.sh
  1254. fi
  1255. for ip in $sub_nodes; do
  1256. log "Bootstrapping $ip"
  1257. # Run overcloud full bootstrap script
  1258. log "Running bootstrap-overcloud-full.sh on $ip"
  1259. ssh $SSH_OPTIONS -t -i /etc/nodepool/id_rsa $ip \
  1260. TRIPLEO_ROOT=$TRIPLEO_ROOT \
  1261. $TRIPLEO_ROOT/tripleo-ci/scripts/$bootstrap_script
  1262. done
  1263. log "Bootstrap subnodes - DONE".
  1264. }
  1265. function setup_nodepool_files {
  1266. log "Setup nodepool files"
  1267. clone openstack-dev/devstack
  1268. clone openstack-infra/devstack-gate
  1269. clone openstack-infra/tripleo-ci
  1270. if [ ! -d $BASE/new ]; then
  1271. ln -s $TRIPLEO_ROOT $BASE/new
  1272. fi
  1273. sudo mkdir -p /etc/nodepool
  1274. sudo chown -R $(whoami): /etc/nodepool
  1275. if [ ! -f /etc/nodepool/id_rsa ]; then
  1276. ssh-keygen -N "" -t rsa -f /etc/nodepool/id_rsa
  1277. cat /etc/nodepool/id_rsa.pub >> ~/.ssh/authorized_keys
  1278. fi
  1279. if [ -z $PRIMARY_NODE_IP ]; then
  1280. echo '$PRIMARY_NODE_IP must be defined. Exiting.'
  1281. exit 1
  1282. fi
  1283. echo $PRIMARY_NODE_IP > /etc/nodepool/primary_node
  1284. echo $PRIMARY_NODE_IP > /etc/nodepool/primary_node_private
  1285. echo -n > /etc/nodepool/sub_nodes
  1286. echo -n > /etc/nodepool/sub_nodes_private
  1287. for sub_node_ip in $SUB_NODE_IPS; do
  1288. echo $sub_node_ip >> /etc/nodepool/sub_nodes
  1289. echo $sub_node_ip >> /etc/nodepool/sub_nodes_private
  1290. done
  1291. for sub_node_ip in $SUB_NODE_IPS; do
  1292. echo $sub_node_ip > /etc/nodepool/node
  1293. echo $sub_node_ip > /etc/nodepool/node_private
  1294. ssh $SSH_OPTIONS -tt $sub_node_ip sudo mkdir -p $TRIPLEO_ROOT
  1295. ssh $SSH_OPTIONS -tt $sub_node_ip sudo chown -R $(whoami): $TRIPLEO_ROOT
  1296. rsync -e "ssh $SSH_OPTIONS" -avhP $TRIPLEO_ROOT $sub_node_ip:$TRIPLEO_ROOT/..
  1297. rsync -e "ssh $SSH_OPTIONS" -avhP /etc/nodepool $sub_node_ip:
  1298. ssh $SSH_OPTIONS -tt $sub_node_ip sudo cp -r nodepool /etc
  1299. ssh $SSH_OPTIONS $sub_node_ip \
  1300. "/bin/bash -c 'cat /etc/nodepool/id_rsa.pub >> ~/.ssh/authorized_keys'"
  1301. done
  1302. echo $PRIMARY_NODE_IP > /etc/nodepool/node
  1303. echo $PRIMARY_NODE_IP > /etc/nodepool/node_private
  1304. log "Setup nodepool files - DONE"
  1305. }
  1306. function bootstrap_subnodes_repos {
  1307. log "Bootstrap subnodes repos"
  1308. local sub_nodes
  1309. sub_nodes=$(cat /etc/nodepool/sub_nodes_private)
  1310. for ip in $sub_nodes; do
  1311. log "Bootstrapping $ip"
  1312. log "Running --repo-setup on $ip"
  1313. # Do repo setup
  1314. # if UPGRADE_RELEASE is set, then we are making an upgrade, so
  1315. # we need to set the stable_release.
  1316. if [ ! -z $UPGRADE_RELEASE ]; then
  1317. log "Stable release $UPGRADE_RELEASE"
  1318. ssh $SSH_OPTIONS -t -i /etc/nodepool/id_rsa $ip \
  1319. "TRIPLEO_ROOT=$TRIPLEO_ROOT; \
  1320. unset STABLE_RELEASE; \
  1321. export STABLE_RELEASE=$UPGRADE_RELEASE; \
  1322. $TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --repo-setup"
  1323. else
  1324. ssh $SSH_OPTIONS -t -i /etc/nodepool/id_rsa $ip \
  1325. TRIPLEO_ROOT=$TRIPLEO_ROOT \
  1326. $TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --repo-setup
  1327. fi
  1328. done
  1329. log "Bootstrap subnodes repos - DONE".
  1330. }
  1331. if [ "$REPO_SETUP" = 1 ]; then
  1332. repo_setup
  1333. fi
  1334. if [ "$DELOREAN_SETUP" = 1 ]; then
  1335. delorean_setup
  1336. fi
  1337. if [ "$DELOREAN_BUILD" = 1 ]; then
  1338. export DELOREAN_BUILD_REFS="${DELOREAN_BUILD_REFS:-$@}"
  1339. if [ -z "$DELOREAN_BUILD_REFS" ]; then
  1340. echo "Usage: $0 --delorean-build openstack/heat openstack/nova"
  1341. exit 1
  1342. fi
  1343. delorean_build
  1344. fi
  1345. if [ "$UNDERCLOUD" = 1 ]; then
  1346. undercloud
  1347. if [ "$UNDERCLOUD_SANITY_CHECK" = 1 ]; then
  1348. undercloud_sanity_check
  1349. fi
  1350. fi
  1351. if [ "$OVERCLOUD_IMAGES" = 1 ]; then
  1352. overcloud_images
  1353. fi
  1354. if [ "$REGISTER_NODES" = 1 ]; then
  1355. register_nodes
  1356. fi
  1357. if [ "$INTROSPECT_NODES" = 1 ]; then
  1358. introspect_nodes
  1359. fi
  1360. if [ "$OVERCLOUD_DEPLOY" = 1 ]; then
  1361. overcloud_deploy
  1362. fi
  1363. if [ "$OVERCLOUD_UPDATE" = 1 ]; then
  1364. overcloud_update
  1365. fi
  1366. if [ "$OVERCLOUD_UPGRADE" = 1 ]; then
  1367. overcloud_upgrade
  1368. fi
  1369. if [ "$OVERCLOUD_UPGRADE_CONVERGE" = 1 ]; then
  1370. overcloud_upgrade_converge
  1371. fi
  1372. if [ "$OVERCLOUD_DELETE" = 1 ]; then
  1373. overcloud_delete
  1374. fi
  1375. if [[ "$USE_CONTAINERS" == 1 && "$OVERCLOUD_DEPLOY" != 1 ]]; then
  1376. echo "Error: --overcloud-deploy flag is required with the flag --use-containers"
  1377. exit 1
  1378. fi
  1379. if [ "$OVERCLOUD_PINGTEST" = 1 ]; then
  1380. overcloud_pingtest
  1381. fi
  1382. if [ "$OVERCLOUD_SANITYTEST" = 1 ]; then
  1383. overcloud_sanitytest
  1384. fi
  1385. if [ "$TEMPEST_RUN" = 1 ]; then
  1386. tempest_run
  1387. fi
  1388. if [ "$UNDERCLOUD_UPGRADE" = 1 ]; then
  1389. undercloud_upgrade
  1390. if [ "$UNDERCLOUD_SANITY_CHECK" = 1 ]; then
  1391. undercloud_sanity_check
  1392. fi
  1393. fi
  1394. if [ "$MULTINODE_SETUP" = 1 ]; then
  1395. multinode_setup
  1396. fi
  1397. if [ "$BOOTSTRAP_SUBNODES" = 1 ]; then
  1398. bootstrap_subnodes
  1399. fi
  1400. if [ "$SETUP_NODEPOOL_FILES" = 1 ]; then
  1401. setup_nodepool_files
  1402. fi
  1403. if [ "$ALL" = 1 ]; then
  1404. repo_setup
  1405. undercloud
  1406. overcloud_images
  1407. register_nodes
  1408. introspect_nodes
  1409. overcloud_deploy
  1410. fi