System configuration for OpenStack Infrastructure
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

run_all.sh 4.3KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152
  1. #!/bin/bash
  2. # Copyright 2014 Hewlett-Packard Development Company, L.P.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. # If updating the puppet system-config repo or installing puppet modules
  16. # fails then abort the puppet run as we will not get the results we
  17. # expect.
  18. set -e
  19. SYSTEM_CONFIG=/opt/system-config
  20. ANSIBLE_PLAYBOOKS=$SYSTEM_CONFIG/playbooks
  21. # We only send stats if running under cron
  22. UNDER_CRON=0
  23. while getopts ":c" arg; do
  24. case $arg in
  25. c)
  26. UNDER_CRON=1
  27. ;;
  28. esac
  29. done
  30. GLOBAL_START_TIME=$(date '+%s')
  31. # Send a timer stat to statsd
  32. # send_timer metric [start_time]
  33. # * uses timer metric bridge.ansible.run_all.<$1>
  34. # * time will be taken from last call of start_timer, or $2 if set
  35. function send_timer {
  36. # Only send stats under cron conditions
  37. if [[ ${UNDER_CRON} != 1 ]]; then
  38. return
  39. fi
  40. local current
  41. current=$(date '+%s')
  42. local name
  43. name=$1
  44. local start
  45. start=${2-$_START_TIME}
  46. local elapsed_ms
  47. elapsed_ms=$(( (current - start) * 1000 ))
  48. echo "bridge.ansible.run_all.${name}:${elapsed_ms}|ms" | nc -w 1 -u graphite.opendev.org 8125
  49. echo "End $name"
  50. }
  51. # See send_timer
  52. function start_timer {
  53. _START_TIME=$(date '+%s')
  54. }
  55. echo "--- begin run @ $(date -Is) ---"
  56. # It's possible for connectivity to a server or manifest application to break
  57. # for indeterminate periods of time, so the playbooks should be run without
  58. # errexit
  59. set +e
  60. # Run all the ansible playbooks under timeout to prevent them from getting
  61. # stuck if they are oomkilled
  62. # Clone system-config and install modules and roles
  63. start_timer
  64. timeout -k 2m 10m ansible-playbook ${ANSIBLE_PLAYBOOKS}/update-system-config.yaml
  65. send_timer update_system_config
  66. # Update the code on bridge
  67. start_timer
  68. timeout -k 2m 10m ansible-playbook ${ANSIBLE_PLAYBOOKS}/bridge.yaml
  69. send_timer bridge
  70. # Run the base playbook everywhere
  71. start_timer
  72. timeout -k 2m 120m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/base.yaml
  73. send_timer base
  74. # Service playbooks
  75. start_timer
  76. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-bridge.yaml
  77. send_timer service-bridge
  78. start_timer
  79. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-gitea-lb.yaml
  80. send_timer gitea-lb
  81. start_timer
  82. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-letsencrypt.yaml
  83. send_timer letsencrypt
  84. start_timer
  85. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-nameserver.yaml
  86. send_timer nameserver
  87. start_timer
  88. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-nodepool.yaml
  89. send_timer nodepool
  90. start_timer
  91. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-mirror-update.yaml
  92. send_timer mirror-update
  93. start_timer
  94. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-mirror.yaml
  95. send_timer mirror
  96. start_timer
  97. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-backup.yaml
  98. send_timer backup
  99. start_timer
  100. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-registry.yaml
  101. send_timer registry
  102. start_timer
  103. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-zuul.yaml
  104. send_timer zuul
  105. # Run the git/gerrit/zuul sequence, since it's important that they all work together
  106. start_timer
  107. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/remote_puppet_git.yaml
  108. send_timer git
  109. # Run AFS changes separately so we can make sure to only do one at a time
  110. # (turns out quorum is nice to have)
  111. start_timer
  112. timeout -k 2m 30m ansible-playbook -f 1 ${ANSIBLE_PLAYBOOKS}/remote_puppet_afs.yaml
  113. send_timer afs
  114. # Run everything else. We do not care if the other things worked
  115. start_timer
  116. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/remote_puppet_else.yaml
  117. send_timer else
  118. # Send the combined time for everything
  119. send_timer total $GLOBAL_START_TIME
  120. echo "--- end run @ $(date -Is) ---"
  121. echo