Collection of scripts and manifests for module testing
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

run_tests.sh 15KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392
  1. #!/bin/bash -ex
  2. # Copyright 2015 Red Hat, Inc.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. export SCRIPT_DIR=$(cd `dirname $0` && pwd -P)
  16. source ${SCRIPT_DIR}/functions
  17. export WORKSPACE=${WORKSPACE:-/tmp}
  18. export CEPH_VERSION=${CEPH_VERSION:-mimic}
  19. export PUPPET_MAJ_VERSION=${PUPPET_MAJ_VERSION:-5}
  20. export SCENARIO=${SCENARIO:-scenario001}
  21. export MANAGE_PUPPET_MODULES=${MANAGE_PUPPET_MODULES:-true}
  22. export MANAGE_REPOS=${MANAGE_REPOS:-true}
  23. export ADD_SWAP=${ADD_SWAP:-true}
  24. export SWAP_SIZE_GB=${SWAP_SIZE_GB:-4}
  25. export HIERA_CONFIG=${HIERA_CONFIG:-${SCRIPT_DIR}/hiera.yaml}
  26. export MANAGE_HIERA=${MANAGE_HIERA:-true}
  27. export PUPPET_ARGS="${PUPPET_ARGS} --detailed-exitcodes --color=false --test --summarize --trace --hiera_config ${HIERA_CONFIG} --logdest ${WORKSPACE}/puppet.log"
  28. export DISTRO=$(lsb_release -c -s)
  29. # If openstack/tempest is broken on master, we can pin the repository to a specific commit
  30. # by using the following line:
  31. if uses_debs; then
  32. # Pin tempest until https://review.openstack.org/#/c/605851/ is included in ubuntu's keystone package
  33. export TEMPEST_VERSION=${TEMPEST_VERSION:-'a7c4c14493d9c20fb89daa5e094d6396a6cbe80d'}
  34. else
  35. export TEMPEST_VERSION=${TEMPEST_VERSION:-'master'}
  36. fi
  37. # For installing Tempest from RPM keep TEMPEST_FROM_SOURCE to false
  38. # In Ubuntu, Tempest packages are not maintained so installing from source
  39. if [ $(lsb_release --id -s) = "Ubuntu" ]; then
  40. export TEMPEST_FROM_SOURCE=${TEMPEST_FROM_SOURCE:-true}
  41. else
  42. export TEMPEST_FROM_SOURCE=${TEMPEST_FROM_SOURCE:-false}
  43. fi
  44. # Cirros Image directory
  45. export IMG_DIR=${IMG_DIR:-'/tmp/openstack/image'}
  46. # if we're running the tests we don't need to write out the facts to facter
  47. # so we can disable it.
  48. export WRITE_FACTS=false
  49. source ${SCRIPT_DIR}/configure_facts.sh
  50. export PATH=${PATH}:/opt/puppetlabs/bin:/opt/puppetlabs/puppet/bin
  51. export PUPPET_BASE_PATH=/etc/puppetlabs/code
  52. export PUPPET_PKG=${PUPPET_PKG:-puppet-agent}
  53. print_header 'Start (run_tests.sh)'
  54. if [ ! -f fixtures/${SCENARIO}.pp ]; then
  55. echo "fixtures/${SCENARIO}.pp file does not exist. Please define a valid scenario."
  56. exit 1
  57. fi
  58. if [ $(id -u) != 0 ]; then
  59. # preserve environment so we can have ZUUL_* params
  60. export SUDO='sudo -E'
  61. fi
  62. if [ "${ADD_SWAP}" = true ]; then
  63. print_header "Create $SWAP_SIZE_GB GB swapfile"
  64. set +e
  65. $SUDO swapon -s |grep -q '/swapfile'
  66. RESULT=$?
  67. set -e
  68. if [ $RESULT -eq 0 ]; then
  69. $SUDO swapoff /swapfile && $SUDO rm -f /swapfile
  70. fi
  71. $SUDO dd if=/dev/zero of=/swapfile count=${SWAP_SIZE_GB}k bs=1M
  72. $SUDO chmod 0600 /swapfile
  73. $SUDO mkswap /swapfile
  74. $SUDO swapon /swapfile
  75. fi
  76. # We install some gems as root so to take benefit of
  77. # OpenStack Infra mirrors.
  78. if [ -f ~/.gemrc ]; then
  79. cat ~/.gemrc | $SUDO tee /root/.gemrc
  80. fi
  81. # handle umask issue after "pam" new release, this is needed when run_tests.sh
  82. # is run remotely via ansible using a user which doesn't have .bashrc file
  83. if [ -f /etc/fedora-release -a -f /etc/bashrc ]; then
  84. source /etc/bashrc
  85. fi
  86. print_header 'Clone Tempest, plugins & pre-cache CirrOS'
  87. # TODO(pabelanger): Move this into tools/install_tempest.sh and add logic so we
  88. # can clone tempest outside of the gate. Also, tempest should be sandboxed into
  89. # the local directory but works needs to be added into puppet to properly find
  90. # the path.
  91. # If it's running in tempest gate, we need to deploy it from source to test the
  92. # patch.
  93. if [[ "${ZUUL_PROJECT}" = "openstack/tempest" ]]; then
  94. TEMPEST_FROM_SOURCE=true
  95. fi
  96. if [ -e /usr/zuul-env/bin/zuul-cloner ] ; then
  97. # For ubuntu we always need to deploy tempest-horizon from source
  98. if uses_debs; then
  99. /usr/zuul-env/bin/zuul-cloner --workspace /tmp --cache-dir /opt/git \
  100. https://git.openstack.org openstack/tempest-horizon
  101. fi
  102. if [ "${TEMPEST_FROM_SOURCE}" = true ]; then
  103. /usr/zuul-env/bin/zuul-cloner --workspace /tmp --cache-dir /opt/git \
  104. https://git.openstack.org openstack/tempest
  105. # Pin Tempest to TEMPEST_VERSION unless we're running inside the
  106. # openstack/tempest gate.
  107. if [[ "${ZUUL_PROJECT}" != "openstack/tempest" ]]; then
  108. pushd /tmp/openstack/tempest
  109. git reset --hard $TEMPEST_VERSION
  110. popd
  111. fi
  112. fi
  113. else
  114. # For ubuntu we always need to deploy tempest-horizon from source
  115. if uses_debs; then
  116. $SUDO rm -rf /tmp/openstack/tempest-horizon
  117. git clone https://git.openstack.org/openstack/tempest-horizon /tmp/openstack/tempest-horizon
  118. fi
  119. if [ "${TEMPEST_FROM_SOURCE}" = true ]; then
  120. $SUDO rm -rf /tmp/openstack/tempest
  121. git clone https://git.openstack.org/openstack/tempest /tmp/openstack/tempest
  122. pushd /tmp/openstack/tempest
  123. git reset --hard origin/$TEMPEST_VERSION
  124. popd
  125. fi
  126. fi
  127. # NOTE(pabelanger): We cache cirros images on our jenkins slaves, check if it
  128. # exists.
  129. if [[ ! -e $IMG_DIR ]]; then
  130. mkdir -p $IMG_DIR
  131. fi
  132. if [ -f ~/cache/files/cirros-0.4.0-x86_64-disk.img ]; then
  133. # Create a symlink for tempest.
  134. if ! [ -h /tmp/openstack/image/cirros-0.4.0-x86_64-disk.img ] ; then
  135. ln -s ~/cache/files/cirros-0.4.0-x86_64-disk.img $IMG_DIR
  136. fi
  137. else
  138. wget http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img -P $IMG_DIR
  139. fi
  140. install_puppet
  141. PUPPET_FULL_PATH=$(which puppet)
  142. if [ "${MANAGE_HIERA}" = true ]; then
  143. configure_hiera
  144. fi
  145. if uses_debs; then
  146. $SUDO apt-get install -y dstat ebtables iotop sysstat
  147. elif is_fedora; then
  148. $SUDO $YUM install -y dstat setools setroubleshoot audit iotop sysstat
  149. $SUDO service auditd start
  150. # SElinux in permissive mode so later we can catch alerts
  151. $SUDO selinuxenabled && $SUDO setenforce 0
  152. fi
  153. # use dstat to monitor system activity during integration testing
  154. if type "dstat" 2>/dev/null; then
  155. print_header 'Start dstat'
  156. $SUDO dstat -tcmndrylpg --top-cpu-adv --top-io-adv --nocolor | $SUDO tee --append /var/log/dstat.log > /dev/null &
  157. fi
  158. if type "iostat" 2>/dev/null; then
  159. print_header 'Start iostat'
  160. $SUDO iostat -x -k -d -t 4 | $SUDO tee --append /var/log/iostat.log > /dev/null &
  161. fi
  162. if [ -f "/usr/sbin/iotop" ]; then
  163. print_header 'Start iotop'
  164. $SUDO /usr/sbin/iotop --kilobytes --only --batch --time --delay=2 --processes --quiet | $SUDO tee --append /var/log/iotop.log > /dev/null &
  165. fi
  166. if [ "${MANAGE_PUPPET_MODULES}" = true ]; then
  167. $SUDO ./install_modules.sh
  168. fi
  169. # Added tempest specific values to common.yaml
  170. if [ "${TEMPEST_FROM_SOURCE}" = false ]; then
  171. echo "tempest::install_from_source: false" >> ${SCRIPT_DIR}/hiera/common.yaml
  172. fi
  173. # Run puppet and assert something changes.
  174. set +e
  175. if [ "${MANAGE_REPOS}" = true ]; then
  176. print_header 'Install repos'
  177. $SUDO $PUPPET_FULL_PATH apply $PUPPET_ARGS -e "include ::openstack_integration::repos"
  178. RESULT=$?
  179. if [ $RESULT -ne 0 ] && [ $RESULT -ne 2 ]; then
  180. print_header 'Puppet failed to install repositories.'
  181. exit 1
  182. fi
  183. print_header 'Updating packages'
  184. if is_fedora; then
  185. $SUDO $YUM update -y
  186. update_ret=$?
  187. elif uses_debs; then
  188. export DEBIAN_FRONTEND=noninteractive
  189. $SUDO apt-get -y -o Dpkg::Options::="--force-confnew" upgrade
  190. update_ret=$?
  191. fi
  192. if [ $update_ret -ne 0 ]; then
  193. print_header 'Error updating packages'
  194. exit 1
  195. fi
  196. fi
  197. print_header "Running Puppet Scenario: ${SCENARIO} (1st time)"
  198. run_puppet $SCENARIO
  199. RESULT=$?
  200. set -e
  201. if [ $RESULT -ne 0 ] && [ $RESULT -ne 2 ]; then
  202. print_header 'First Puppet run contains errors in catalog.'
  203. catch_puppet_failures
  204. print_header 'SELinux Alerts (1st time)'
  205. catch_selinux_alerts
  206. exit 1
  207. fi
  208. timestamp_puppet_log
  209. # Run puppet a second time and assert nothing changes.
  210. set +e
  211. print_header "Running Puppet Scenario: ${SCENARIO} (2nd time)"
  212. run_puppet $SCENARIO
  213. RESULT=$?
  214. set -e
  215. if [ $RESULT -ne 0 ]; then
  216. print_header 'Second Puppet run is not idempotent.'
  217. catch_puppet_failures
  218. print_header 'SELinux Alerts (2nd time)'
  219. catch_selinux_alerts
  220. exit 1
  221. fi
  222. timestamp_puppet_log
  223. print_header 'Prepare Tempest'
  224. # FIXME: Since tempest create tempest workspace which is owned by root user.
  225. # We need to fix it in puppet-tempest, as a workaround we are changing the mode
  226. # of tempest workspace and run tempest command using root.
  227. $SUDO touch /tmp/openstack/tempest/test-whitelist.txt /tmp/openstack/tempest/test-blacklist.txt
  228. $SUDO chown -R "$(id -u):$(id -g)" /tmp/openstack/tempest/
  229. # install from source now on ubuntu until packaged
  230. if uses_debs; then
  231. cd /tmp/openstack/tempest-horizon;
  232. if [ $(lsb_release --id -s) = "Debian" ] ; then
  233. $SUDO apt-get install -y tempest python3-stestr python3-os-testr python3-tempest python3-tempest-horizon
  234. else
  235. $SUDO python setup.py install
  236. $SUDO pip install -U stestr os-testr
  237. fi
  238. fi
  239. set +e
  240. # Select what to test:
  241. # Smoke suite
  242. echo "smoke" > /tmp/openstack/tempest/test-whitelist.txt
  243. # Horizon
  244. echo "dashboard" >> /tmp/openstack/tempest/test-whitelist.txt
  245. # Aodh
  246. echo "TelemetryAlarming" >> /tmp/openstack/tempest/test-whitelist.txt
  247. # Gnocchi
  248. echo "telemetry_tempest_plugin.gnocchi" >> /tmp/openstack/tempest/test-whitelist.txt
  249. # Vitrage
  250. echo "TestEvents" >> /tmp/openstack/tempest/test-whitelist.txt
  251. # Test Autoscaling with Telemetry (need panko, ubuntu doesn't ship it)
  252. uses_debs || echo "test_telemetry_integration" >> /tmp/openstack/tempest/test-whitelist.txt
  253. # Ironic
  254. # Note: running all Ironic tests under SSL is not working
  255. # https://bugs.launchpad.net/ironic/+bug/1554237
  256. echo "ironic_tempest_plugin.tests.api.admin.test_drivers" >> /tmp/openstack/tempest/test-whitelist.txt
  257. # Zaqar
  258. echo "v2.test_queues.TestManageQueue" >> /tmp/openstack/tempest/test-whitelist.txt
  259. # ec2api
  260. # VPN tests require VPNaaS, which doesn't work yet in puppet-tempest.
  261. # As soon as enabling neutron_vpnaas_available works there, the VPN tests can
  262. # be included.
  263. echo "ec2api_tempest_plugin.api.*test_create_delete(?!.*_vpn_connection)" >> /tmp/openstack/tempest/test-whitelist.txt
  264. # Cinder Backup
  265. echo "VolumesBackupsAdminV2Test" >> /tmp/openstack/tempest/test-whitelist.txt
  266. # Cinder encrypted volumes
  267. echo "TestEncryptedCinderVolumes" >> /tmp/openstack/tempest/test-whitelist.txt
  268. # Mistral
  269. # We have to ignore a smoke test because of:
  270. # https://bugs.launchpad.net/mistral/+bug/1654555
  271. echo "test_create_and_delete_workflow" >> /tmp/openstack/tempest/test-whitelist.txt
  272. # BGPVPN
  273. echo "test_create_bgpvpn" >> /tmp/openstack/tempest/test-whitelist.txt
  274. # L2GW
  275. echo "test_create_show_list_update_delete_l2gateway" >> /tmp/openstack/tempest/test-whitelist.txt
  276. if uses_debs; then
  277. echo "mistral_tempest_tests.tests.api.v2.test_executions.ExecutionTestsV2.test_get_list_executions" > /tmp/openstack/tempest/test-blacklist.txt
  278. echo "ceilometer.tests.tempest.api.test_telemetry_notification_api.TelemetryNotificationAPITest.test_check_glance_v2_notifications" >> /tmp/openstack/tempest/test-blacklist.txt
  279. echo "tempest_horizon.tests.scenario.test_dashboard_basic_ops.TestDashboardBasicOps.test_basic_scenario" >> /tmp/openstack/tempest/test-blacklist.txt
  280. echo "telemetry_tempest_plugin.scenario.test_telemetry_integration.TestTelemetryIntegration" >> /tmp/openstack/tempest/test-blacklist.txt
  281. # TODO (amoralej) tempest tests for object_storage are not working in master with current version of tempest in uca (16.1.0).
  282. echo "tempest.api.object_storage" >> /tmp/openstack/tempest/test-blacklist.txt
  283. EXCLUDES="--blacklist-file=/tmp/openstack/tempest/test-blacklist.txt"
  284. # TODO(tobias-urdin): We must have the neutron-tempest-plugin to even test Neutron, is also required by
  285. # vpnaas and dynamic routing projects.
  286. $SUDO pip install neutron-tempest-plugin
  287. else
  288. # https://review.openstack.org/#/c/504345/ has changed the behavior of tempest when running with --regex and --whitelist-file
  289. # and now operator between them is OR when filtering tests (which is how it was documented, btw). In order to promote
  290. # we need to remove this regex option and implement https://review.openstack.org/#/c/547278 when ready.
  291. # Note these tests were disabled in https://review.openstack.org/#/c/461969/ and hopefully it's more stable now and allows
  292. # us to run it until we can implement --blacklist-file in a stable way.
  293. #EXCLUDES="--regex=^(?!tempest.scenario.gnocchi.test.live_assert_vcpus_metric_is_really_expurged.test_request.*$)(?!tempest.scenario.gnocchi.test.live_assert_no_delete_metrics_have_the_gabbilive_policy.test_request.*$).*"
  294. # Note(chandankumar): Blacklist tempest_horizon.tests.scenario.test_dashboard_basic_ops test as they are currently flacky in CI on CentOS
  295. # Adding it to skip list will help till we find the correct solution
  296. EXCLUDES="--black-regex=^tempest_horizon.tests.scenario.test_dashboard_basic_ops|telemetry_tempest_plugin.scenario.test_telemetry_integration.TestTelemetryIntegration"
  297. fi
  298. print_header 'Running Tempest'
  299. cd /tmp/openstack/tempest
  300. if [ "${TEMPEST_FROM_SOURCE}" = true ]; then
  301. virtualenv --system-site-packages run_tempest
  302. run_tempest/bin/pip install -c https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt -U -r requirements.txt
  303. run_tempest/bin/python setup.py install
  304. run_tempest/bin/stestr init
  305. export tempest_binary="run_tempest/bin/tempest"
  306. elif [ $(lsb_release --id -s) = "Debian" ]; then
  307. export tempest_binary="/usr/bin/python3-tempest"
  308. else
  309. export tempest_binary="/usr/bin/tempest"
  310. fi
  311. # List tempest version
  312. $tempest_binary --version
  313. # List tempest plugins
  314. $tempest_binary list-plugins
  315. # list tempest workspace
  316. $tempest_binary workspace list
  317. # list tempest tests before running tempest
  318. $tempest_binary run -l --whitelist_file=/tmp/openstack/tempest/test-whitelist.txt
  319. # Run tempest tests
  320. $tempest_binary run --whitelist_file=/tmp/openstack/tempest/test-whitelist.txt --concurrency=2 $EXCLUDES
  321. RESULT=$?
  322. set -e
  323. if [ -d .testrepository ]; then
  324. testr last --subunit > /tmp/openstack/tempest/testrepository.subunit
  325. elif [ -d .stestr ]; then
  326. if type "stestr-3" 2>/dev/null; then
  327. stestr-3 last --subunit > /tmp/openstack/tempest/testrepository.subunit
  328. else
  329. stestr last --subunit > /tmp/openstack/tempest/testrepository.subunit
  330. fi
  331. fi
  332. subunit2html /tmp/openstack/tempest/testrepository.subunit /tmp/openstack/tempest/testr_results.html
  333. print_header 'SELinux Alerts (Tempest)'
  334. catch_selinux_alerts
  335. print_header 'Done (run_tests.sh)'
  336. exit $RESULT