System configuration for OpenStack Infrastructure
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

107 lines
3.0 KiB

  1. #!/bin/bash
  2. # Copyright 2014 Hewlett-Packard Development Company, L.P.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. # If updating the puppet system-config repo or installing puppet modules
  16. # fails then abort the puppet run as we will not get the results we
  17. # expect.
  18. set -e
  19. SYSTEM_CONFIG=/opt/system-config
  20. ANSIBLE_PLAYBOOKS=$SYSTEM_CONFIG/playbooks
  21. # We only send stats if running under cron
  22. UNDER_CRON=0
  23. while getopts ":c" arg; do
  24. case $arg in
  25. c)
  26. UNDER_CRON=1
  27. ;;
  28. esac
  29. done
  30. GLOBAL_START_TIME=$(date '+%s')
  31. # Send a timer stat to statsd
  32. # send_timer metric [start_time]
  33. # * uses timer metric bridge.ansible.run_all.<$1>
  34. # * time will be taken from last call of start_timer, or $2 if set
  35. function send_timer {
  36. # Only send stats under cron conditions
  37. if [[ ${UNDER_CRON} != 1 ]]; then
  38. return
  39. fi
  40. local current
  41. current=$(date '+%s')
  42. local name
  43. name=$1
  44. local start
  45. start=${2-$_START_TIME}
  46. local elapsed_ms
  47. elapsed_ms=$(( (current - start) * 1000 ))
  48. echo "bridge.ansible.run_all.${name}:${elapsed_ms}|ms" | nc -w 1 -u graphite.opendev.org 8125
  49. echo "End $name"
  50. }
  51. # See send_timer
  52. function start_timer {
  53. _START_TIME=$(date '+%s')
  54. }
  55. echo "--- begin run @ $(date -Is) ---"
  56. # It's possible for connectivity to a server or manifest application to break
  57. # for indeterminate periods of time, so the playbooks should be run without
  58. # errexit
  59. set +e
  60. # Run all the ansible playbooks under timeout to prevent them from getting
  61. # stuck if they are oomkilled
  62. start_timer
  63. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-zuul.yaml
  64. send_timer zuul
  65. start_timer
  66. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-review-dev.yaml
  67. send_timer nodepool
  68. start_timer
  69. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-review.yaml
  70. send_timer nodepool
  71. start_timer
  72. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-gitea.yaml
  73. send_timer nodepool
  74. # Run AFS changes separately so we can make sure to only do one at a time
  75. # (turns out quorum is nice to have)
  76. start_timer
  77. timeout -k 2m 30m ansible-playbook -f 1 ${ANSIBLE_PLAYBOOKS}/remote_puppet_afs.yaml
  78. send_timer afs
  79. # Run everything else. We do not care if the other things worked
  80. start_timer
  81. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/remote_puppet_else.yaml
  82. send_timer else
  83. # Send the combined time for everything
  84. send_timer total $GLOBAL_START_TIME
  85. echo "--- end run @ $(date -Is) ---"
  86. echo