Manage a pool of nodes for a distributed test infrastructure
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

check_devstack_plugin.sh 8.0KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264
  1. #!/bin/bash -ex
  2. LOGDIR=$1
  3. # Set to indiciate an error return
  4. RETURN=0
  5. FAILURE_REASON=""
  6. NODEPOOL_INSTALL=${NODEPOOL_INSTALL:-/opt/stack/nodepool-venv}
  7. NODEPOOL_CONFIG=${NODEPOOL_CONFIG:-/etc/nodepool/nodepool.yaml}
  8. NODEPOOL_SECURE=${NODEPOOL_SECURE:-/etc/nodepool/secure.conf}
  9. NODEPOOL="$NODEPOOL_INSTALL/bin/nodepool -c $NODEPOOL_CONFIG -s $NODEPOOL_SECURE"
  10. # Source stackrc so that we get the variables about enabled images set
  11. # from the devstack job. That's the ones we'll wait for below.
  12. if [[ ! -f /opt/stack/devstack/stackrc ]]; then
  13. echo "Can not find enabled images from devstack run!"
  14. exit 1
  15. else
  16. source /opt/stack/devstack/stackrc
  17. fi
  18. NODEPOOL_PAUSE_CENTOS_7_DIB=${NODEPOOL_PAUSE_CENTOS_7_DIB:-True}
  19. NODEPOOL_PAUSE_DEBIAN_STRETCH_DIB=${NODEPOOL_PAUSE_DEBIAN_STRETCH_DIB:-True}
  20. NODEPOOL_PAUSE_FEDORA_29_DIB=${NODEPOOL_PAUSE_FEDORA_29_DIB:-True}
  21. NODEPOOL_PAUSE_UBUNTU_BIONIC_DIB=${NODEPOOL_PAUSE_UBUNTU_BIONIC_DIB:-True}
  22. NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB=${NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB:-True}
  23. NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB=${NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB:-True}
  24. NODEPOOL_PAUSE_OPENSUSE_423_DIB=${NODEPOOL_PAUSE_OPENSUSE_423_DIB:-True}
  25. NODEPOOL_PAUSE_OPENSUSE_150_DIB=${NODEPOOL_PAUSE_OPENSUSE_150_DIB:-True}
  26. NODEPOOL_PAUSE_OPENSUSE_TUMBLEWEED_DIB=${NODEPOOL_PAUSE_OPENSUSE_TUMBLEWEED_DIB:-True}
  27. NODEPOOL_PAUSE_GENTOO_17_0_SYSTEMD_DIB=${NODEPOOL_PAUSE_GENTOO_17_0_SYSTEMD_DIB:-True}
  28. function sshintonode {
  29. name=$1
  30. state='ready'
  31. node=`$NODEPOOL list | grep $name | grep $state | cut -d '|' -f6 | tr -d ' '`
  32. /tmp/ssh_wrapper $node ls /
  33. # Check that the root partition grew on boot; it should be a 5GiB
  34. # partition minus some space for the boot partition. However
  35. # emperical evidence suggests there is some modulo maths going on,
  36. # (possibly with alignment?) that means we can vary up to even
  37. # 64MiB. Thus we choose an expected value that gives us enough
  38. # slop to avoid false matches, but still indicates we resized up.
  39. root_size=$(/tmp/ssh_wrapper $node -- lsblk -rbno SIZE /dev/vda1)
  40. expected_root_size=$(( 5000000000 ))
  41. if [[ $root_size -lt $expected_root_size ]]; then
  42. echo "*** Root device does not appear to have grown: $root_size"
  43. FAILURE_REASON="Root partition of $name does not appear to have grown: $root_size < $expected_root_size"
  44. RETURN=1
  45. fi
  46. # Check we saw metadata deployed to the config-drive
  47. /tmp/ssh_wrapper $node \
  48. "dd status=none if=/dev/sr0 | tr -cd '[:print:]' | grep -q nodepool_devstack"
  49. if [[ $? -ne 0 ]]; then
  50. echo "*** Failed to find metadata in config-drive"
  51. FAILURE_REASON="Failed to find meta-data in config-drive for $node"
  52. RETURN=1
  53. fi
  54. }
  55. function showserver {
  56. name=$1
  57. state='ready'
  58. node_id=`$NODEPOOL list | grep $name | grep $state | cut -d '|' -f5 | tr -d ' '`
  59. EXPECTED=$(mktemp)
  60. RESULT=$(mktemp)
  61. source /opt/stack/devstack/openrc admin admin
  62. nova show $node_id | grep -Eo "user_data[ ]+.*|[ ]*$" | awk {'print $3'} |\
  63. base64 --decode > $RESULT
  64. cat <<EOF >$EXPECTED
  65. #cloud-config
  66. write_files:
  67. - content: |
  68. testpassed
  69. path: /etc/testfile_nodepool_userdata
  70. EOF
  71. diff $EXPECTED $RESULT
  72. if [[ $? -ne 0 ]]; then
  73. echo "*** Failed to find userdata on server!"
  74. FAILURE_REASON="Failed to find userdata on server for $node"
  75. echo "Expected userdata:"
  76. cat $EXPECTED
  77. echo "Found userdata:"
  78. cat $RESULT
  79. RETURN=1
  80. fi
  81. }
  82. function checknm {
  83. name=$1
  84. state='ready'
  85. node=`$NODEPOOL list | grep $name | grep $state | cut -d '|' -f6 | tr -d ' '`
  86. nm_output=$(/tmp/ssh_wrapper $node -- nmcli c)
  87. # virtio device is eth0 on older, ens3 on newer
  88. if [[ ! ${nm_output} =~ (eth0|ens3) ]]; then
  89. echo "*** Failed to find interface in NetworkManager connections"
  90. /tmp/ssh_wrapper $node -- nmcli c
  91. /tmp/ssh_wrapper $node -- nmcli device
  92. FAILURE_REASON="Failed to find interface in NetworkManager connections"
  93. RETURN=1
  94. fi
  95. }
  96. function waitforimage {
  97. local name=$1
  98. local state='ready'
  99. local builds
  100. while ! $NODEPOOL image-list | grep $name | grep $state; do
  101. $NODEPOOL image-list > ${LOGDIR}/nodepool-image-list.txt
  102. $NODEPOOL list --detail > ${LOGDIR}/nodepool-list.txt
  103. builds=$(ls -l /var/log/nodepool/builds/ | grep $name | wc -l)
  104. if [[ ${builds} -ge 4 ]]; then
  105. echo "*** Build of $name failed at least 3 times, aborting"
  106. exit 1
  107. fi
  108. sleep 10
  109. done
  110. }
  111. function waitfornode {
  112. name=$1
  113. state='ready'
  114. while ! $NODEPOOL list | grep $name | grep $state | grep "unlocked"; do
  115. $NODEPOOL image-list > ${LOGDIR}/nodepool-image-list.txt
  116. $NODEPOOL list --detail > ${LOGDIR}/nodepool-list.txt
  117. sleep 10
  118. done
  119. }
  120. if [ ${NODEPOOL_PAUSE_CENTOS_7_DIB,,} = 'false' ]; then
  121. # check that image built
  122. waitforimage centos-7
  123. # check image was bootable
  124. waitfornode centos-7
  125. # check ssh for root user
  126. sshintonode centos-7
  127. # networkmanager check
  128. checknm centos-7
  129. # userdata check
  130. showserver centos-7
  131. fi
  132. if [ ${NODEPOOL_PAUSE_DEBIAN_STRETCH_DIB,,} = 'false' ]; then
  133. # check that image built
  134. waitforimage debian-stretch
  135. # check image was bootable
  136. waitfornode debian-stretch
  137. # check ssh for root user
  138. sshintonode debian-stretch
  139. # userdata check
  140. showserver debian-stretch
  141. fi
  142. if [ ${NODEPOOL_PAUSE_FEDORA_29_DIB,,} = 'false' ]; then
  143. # check that image built
  144. waitforimage fedora-29
  145. # check image was bootable
  146. waitfornode fedora-29
  147. # check ssh for root user
  148. sshintonode fedora-29
  149. # networkmanager check
  150. checknm fedora-29
  151. # userdata check
  152. showserver fedora-29
  153. fi
  154. if [ ${NODEPOOL_PAUSE_UBUNTU_BIONIC_DIB,,} = 'false' ]; then
  155. # check that image built
  156. waitforimage ubuntu-bionic
  157. # check image was bootable
  158. waitfornode ubuntu-bionic
  159. # check ssh for root user
  160. sshintonode ubuntu-bionic
  161. # userdata check
  162. showserver ubuntu-bionic
  163. fi
  164. if [ ${NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB,,} = 'false' ]; then
  165. # check that image built
  166. waitforimage ubuntu-trusty
  167. # check image was bootable
  168. waitfornode ubuntu-trusty
  169. # check ssh for root user
  170. sshintonode ubuntu-trusty
  171. # userdata check
  172. showserver ubuntu-trusty
  173. fi
  174. if [ ${NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB,,} = 'false' ]; then
  175. # check that image built
  176. waitforimage ubuntu-xenial
  177. # check image was bootable
  178. waitfornode ubuntu-xenial
  179. # check ssh for root user
  180. sshintonode ubuntu-xenial
  181. # userdata check
  182. showserver ubuntu-xenial
  183. fi
  184. if [ ${NODEPOOL_PAUSE_OPENSUSE_423_DIB,,} = 'false' ]; then
  185. # check that image built
  186. waitforimage opensuse-423
  187. # check image was bootable
  188. waitfornode opensuse-423
  189. # check ssh for root user
  190. sshintonode opensuse-423
  191. # userdata check
  192. showserver opensuse-423
  193. fi
  194. if [ ${NODEPOOL_PAUSE_OPENSUSE_150_DIB,,} = 'false' ]; then
  195. # check that image built
  196. waitforimage opensuse-150
  197. # check image was bootable
  198. waitfornode opensuse-150
  199. # check ssh for root user
  200. sshintonode opensuse-150
  201. # userdata check
  202. showserver opensuse-150
  203. fi
  204. if [ ${NODEPOOL_PAUSE_OPENSUSE_TUMBLEWEED_DIB,,} = 'false' ]; then
  205. # check that image built
  206. waitforimage opensuse-tumbleweed
  207. # check image was bootable
  208. waitfornode opensuse-tumbleweed
  209. # check ssh for root user
  210. sshintonode opensuse-tumbleweed
  211. # userdata check
  212. showserver opensuse-tumbleweed
  213. fi
  214. if [ ${NODEPOOL_PAUSE_GENTOO_17_0_SYSTEMD_DIB,,} = 'false' ]; then
  215. # check that image built
  216. waitforimage gentoo-17-0-systemd
  217. # check image was bootable
  218. waitfornode gentoo-17-0-systemd
  219. # check ssh for root user
  220. sshintonode gentoo-17-0-systemd
  221. # userdata check
  222. showserver gentoo-17-0-systemd
  223. fi
  224. set -o errexit
  225. # Show the built nodes
  226. $NODEPOOL list
  227. # Try to delete the nodes that were just built
  228. $NODEPOOL delete --now 0000000000
  229. # show the deleted nodes (and their replacements may be building)
  230. $NODEPOOL list
  231. if [[ -n "${FAILURE_REASON}" ]]; then
  232. echo "${FAILURE_REASON}"
  233. fi
  234. exit $RETURN