Manage a pool of nodes for a distributed test infrastructure
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

check_devstack_plugin.sh 6.8KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214
  1. #!/bin/bash -ex
  2. LOGDIR=$1
  3. # Set to indiciate an error return
  4. RETURN=0
  5. FAILURE_REASON=""
  6. NODEPOOL_INSTALL=${NODEPOOL_INSTALL:-/opt/stack/nodepool-venv}
  7. NODEPOOL_CONFIG=${NODEPOOL_CONFIG:-/etc/nodepool/nodepool.yaml}
  8. NODEPOOL_SECURE=${NODEPOOL_SECURE:-/etc/nodepool/secure.conf}
  9. NODEPOOL="$NODEPOOL_INSTALL/bin/nodepool -c $NODEPOOL_CONFIG -s $NODEPOOL_SECURE"
  10. # Source stackrc so that we get the variables about enabled images set
  11. # from the devstack job. That's the ones we'll wait for below.
  12. if [[ ! -f /opt/stack/devstack/stackrc ]]; then
  13. echo "Can not find enabled images from devstack run!"
  14. exit 1
  15. else
  16. source /opt/stack/devstack/stackrc
  17. fi
  18. NODEPOOL_PAUSE_CENTOS_7_DIB=${NODEPOOL_PAUSE_CENTOS_7_DIB:-True}
  19. NODEPOOL_PAUSE_DEBIAN_STRETCH_DIB=${NODEPOOL_PAUSE_DEBIAN_STRETCH_DIB:-True}
  20. NODEPOOL_PAUSE_FEDORA_29_DIB=${NODEPOOL_PAUSE_FEDORA_29_DIB:-True}
  21. NODEPOOL_PAUSE_UBUNTU_BIONIC_DIB=${NODEPOOL_PAUSE_UBUNTU_BIONIC_DIB:-True}
  22. NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB=${NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB:-True}
  23. NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB=${NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB:-True}
  24. NODEPOOL_PAUSE_OPENSUSE_423_DIB=${NODEPOOL_PAUSE_OPENSUSE_423_DIB:-True}
  25. NODEPOOL_PAUSE_OPENSUSE_150_DIB=${NODEPOOL_PAUSE_OPENSUSE_150_DIB:-True}
  26. NODEPOOL_PAUSE_OPENSUSE_TUMBLEWEED_DIB=${NODEPOOL_PAUSE_OPENSUSE_TUMBLEWEED_DIB:-True}
  27. NODEPOOL_PAUSE_GENTOO_17_0_SYSTEMD_DIB=${NODEPOOL_PAUSE_GENTOO_17_0_SYSTEMD_DIB:-True}
  28. function sshintonode {
  29. name=$1
  30. state='ready'
  31. node=`$NODEPOOL list | grep $name | grep $state | cut -d '|' -f6 | tr -d ' '`
  32. /tmp/ssh_wrapper $node ls /
  33. # Check that the root partition grew on boot; it should be a 5GiB
  34. # partition minus some space for the boot partition. However
  35. # emperical evidence suggests there is some modulo maths going on,
  36. # (possibly with alignment?) that means we can vary up to even
  37. # 64MiB. Thus we choose an expected value that gives us enough
  38. # slop to avoid false matches, but still indicates we resized up.
  39. root_size=$(/tmp/ssh_wrapper $node -- lsblk -rbno SIZE /dev/vda1)
  40. expected_root_size=$(( 5000000000 ))
  41. if [[ $root_size -lt $expected_root_size ]]; then
  42. echo "*** Root device does not appear to have grown: $root_size"
  43. FAILURE_REASON="Root partition of $name does not appear to have grown: $root_size < $expected_root_size"
  44. RETURN=1
  45. fi
  46. # Check we saw metadata deployed to the config-drive
  47. /tmp/ssh_wrapper $node \
  48. "dd status=none if=/dev/sr0 | tr -cd '[:print:]' | grep -q nodepool_devstack"
  49. if [[ $? -ne 0 ]]; then
  50. echo "*** Failed to find metadata in config-drive"
  51. FAILURE_REASON="Failed to find meta-data in config-drive for $node"
  52. RETURN=1
  53. fi
  54. }
  55. function checknm {
  56. name=$1
  57. state='ready'
  58. node=`$NODEPOOL list | grep $name | grep $state | cut -d '|' -f6 | tr -d ' '`
  59. nm_output=$(/tmp/ssh_wrapper $node -- nmcli c)
  60. # virtio device is eth0 on older, ens3 on newer
  61. if [[ ! ${nm_output} =~ (eth0|ens3) ]]; then
  62. echo "*** Failed to find interface in NetworkManager connections"
  63. /tmp/ssh_wrapper $node -- nmcli c
  64. /tmp/ssh_wrapper $node -- nmcli device
  65. FAILURE_REASON="Failed to find interface in NetworkManager connections"
  66. RETURN=1
  67. fi
  68. }
  69. function waitforimage {
  70. local name=$1
  71. local state='ready'
  72. local builds
  73. while ! $NODEPOOL image-list | grep $name | grep $state; do
  74. $NODEPOOL image-list > ${LOGDIR}/nodepool-image-list.txt
  75. $NODEPOOL list --detail > ${LOGDIR}/nodepool-list.txt
  76. builds=$(ls -l /var/log/nodepool/builds/ | grep $name | wc -l)
  77. if [[ ${builds} -ge 4 ]]; then
  78. echo "*** Build of $name failed at least 3 times, aborting"
  79. exit 1
  80. fi
  81. sleep 10
  82. done
  83. }
  84. function waitfornode {
  85. name=$1
  86. state='ready'
  87. while ! $NODEPOOL list | grep $name | grep $state | grep "unlocked"; do
  88. $NODEPOOL image-list > ${LOGDIR}/nodepool-image-list.txt
  89. $NODEPOOL list --detail > ${LOGDIR}/nodepool-list.txt
  90. sleep 10
  91. done
  92. }
  93. if [ ${NODEPOOL_PAUSE_CENTOS_7_DIB,,} = 'false' ]; then
  94. # check that image built
  95. waitforimage centos-7
  96. # check image was bootable
  97. waitfornode centos-7
  98. # check ssh for root user
  99. sshintonode centos-7
  100. # networkmanager check
  101. checknm centos-7
  102. fi
  103. if [ ${NODEPOOL_PAUSE_DEBIAN_STRETCH_DIB,,} = 'false' ]; then
  104. # check that image built
  105. waitforimage debian-stretch
  106. # check image was bootable
  107. waitfornode debian-stretch
  108. # check ssh for root user
  109. sshintonode debian-stretch
  110. fi
  111. if [ ${NODEPOOL_PAUSE_FEDORA_29_DIB,,} = 'false' ]; then
  112. # check that image built
  113. waitforimage fedora-29
  114. # check image was bootable
  115. waitfornode fedora-29
  116. # check ssh for root user
  117. sshintonode fedora-29
  118. # networkmanager check
  119. checknm fedora-29
  120. fi
  121. if [ ${NODEPOOL_PAUSE_UBUNTU_BIONIC_DIB,,} = 'false' ]; then
  122. # check that image built
  123. waitforimage ubuntu-bionic
  124. # check image was bootable
  125. waitfornode ubuntu-bionic
  126. # check ssh for root user
  127. sshintonode ubuntu-bionic
  128. fi
  129. if [ ${NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB,,} = 'false' ]; then
  130. # check that image built
  131. waitforimage ubuntu-trusty
  132. # check image was bootable
  133. waitfornode ubuntu-trusty
  134. # check ssh for root user
  135. sshintonode ubuntu-trusty
  136. fi
  137. if [ ${NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB,,} = 'false' ]; then
  138. # check that image built
  139. waitforimage ubuntu-xenial
  140. # check image was bootable
  141. waitfornode ubuntu-xenial
  142. # check ssh for root user
  143. sshintonode ubuntu-xenial
  144. fi
  145. if [ ${NODEPOOL_PAUSE_OPENSUSE_423_DIB,,} = 'false' ]; then
  146. # check that image built
  147. waitforimage opensuse-423
  148. # check image was bootable
  149. waitfornode opensuse-423
  150. # check ssh for root user
  151. sshintonode opensuse-423
  152. fi
  153. if [ ${NODEPOOL_PAUSE_OPENSUSE_150_DIB,,} = 'false' ]; then
  154. # check that image built
  155. waitforimage opensuse-150
  156. # check image was bootable
  157. waitfornode opensuse-150
  158. # check ssh for root user
  159. sshintonode opensuse-150
  160. fi
  161. if [ ${NODEPOOL_PAUSE_OPENSUSE_TUMBLEWEED_DIB,,} = 'false' ]; then
  162. # check that image built
  163. waitforimage opensuse-tumbleweed
  164. # check image was bootable
  165. waitfornode opensuse-tumbleweed
  166. # check ssh for root user
  167. sshintonode opensuse-tumbleweed
  168. fi
  169. if [ ${NODEPOOL_PAUSE_GENTOO_17_0_SYSTEMD_DIB,,} = 'false' ]; then
  170. # check that image built
  171. waitforimage gentoo-17-0-systemd
  172. # check image was bootable
  173. waitfornode gentoo-17-0-systemd
  174. # check ssh for root user
  175. sshintonode gentoo-17-0-systemd
  176. fi
  177. set -o errexit
  178. # Show the built nodes
  179. $NODEPOOL list
  180. # Try to delete the nodes that were just built
  181. $NODEPOOL delete --now 0000000000
  182. # show the deleted nodes (and their replacements may be building)
  183. $NODEPOOL list
  184. if [[ -n "${FAILURE_REASON}" ]]; then
  185. echo "${FAILURE_REASON}"
  186. fi
  187. exit $RETURN