System configuration for OpenStack Infrastructure
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

157 lines
4.4KB

  1. #!/bin/bash
  2. # Copyright 2014 Hewlett-Packard Development Company, L.P.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. # If updating the puppet system-config repo or installing puppet modules
  16. # fails then abort the puppet run as we will not get the results we
  17. # expect.
  18. set -e
  19. SYSTEM_CONFIG=/opt/system-config
  20. ANSIBLE_PLAYBOOKS=$SYSTEM_CONFIG/playbooks
  21. # We only send stats if running under cron
  22. UNDER_CRON=0
  23. while getopts ":c" arg; do
  24. case $arg in
  25. c)
  26. UNDER_CRON=1
  27. ;;
  28. esac
  29. done
  30. GLOBAL_START_TIME=$(date '+%s')
  31. # Send a timer stat to statsd
  32. # send_timer metric [start_time]
  33. # * uses timer metric bridge.ansible.run_all.<$1>
  34. # * time will be taken from last call of start_timer, or $2 if set
  35. function send_timer {
  36. # Only send stats under cron conditions
  37. if [[ ${UNDER_CRON} != 1 ]]; then
  38. return
  39. fi
  40. local current
  41. current=$(date '+%s')
  42. local name
  43. name=$1
  44. local start
  45. start=${2-$_START_TIME}
  46. local elapsed_ms
  47. elapsed_ms=$(( (current - start) * 1000 ))
  48. echo "bridge.ansible.run_all.${name}:${elapsed_ms}|ms" | nc -w 1 -u graphite.opendev.org 8125
  49. echo "End $name"
  50. }
  51. # See send_timer
  52. function start_timer {
  53. _START_TIME=$(date '+%s')
  54. }
  55. echo "--- begin run @ $(date -Is) ---"
  56. # It's possible for connectivity to a server or manifest application to break
  57. # for indeterminate periods of time, so the playbooks should be run without
  58. # errexit
  59. set +e
  60. # Run all the ansible playbooks under timeout to prevent them from getting
  61. # stuck if they are oomkilled
  62. # Clone system-config and install modules and roles
  63. start_timer
  64. timeout -k 2m 10m ansible-playbook ${ANSIBLE_PLAYBOOKS}/update-system-config.yaml
  65. send_timer update_system_config
  66. # Update the code on bridge
  67. start_timer
  68. timeout -k 2m 10m ansible-playbook ${ANSIBLE_PLAYBOOKS}/bridge.yaml
  69. send_timer bridge
  70. # Run the base playbook everywhere
  71. start_timer
  72. timeout -k 2m 120m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/base.yaml
  73. send_timer base
  74. # Service playbooks
  75. start_timer
  76. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-bridge.yaml
  77. send_timer service-bridge
  78. start_timer
  79. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-gitea-lb.yaml
  80. send_timer gitea-lb
  81. start_timer
  82. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-letsencrypt.yaml
  83. send_timer letsencrypt
  84. start_timer
  85. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-nameserver.yaml
  86. send_timer nameserver
  87. start_timer
  88. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-nodepool.yaml
  89. send_timer nodepool
  90. start_timer
  91. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-mirror-update.yaml
  92. send_timer mirror-update
  93. start_timer
  94. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-mirror.yaml
  95. send_timer mirror
  96. start_timer
  97. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-backup.yaml
  98. send_timer backup
  99. start_timer
  100. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-registry.yaml
  101. send_timer registry
  102. start_timer
  103. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-zuul.yaml
  104. send_timer zuul
  105. start_timer
  106. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-review-dev.yaml
  107. send_timer nodepool
  108. # Run the git/gerrit/zuul sequence, since it's important that they all work together
  109. start_timer
  110. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/remote_puppet_git.yaml
  111. send_timer git
  112. # Run AFS changes separately so we can make sure to only do one at a time
  113. # (turns out quorum is nice to have)
  114. start_timer
  115. timeout -k 2m 30m ansible-playbook -f 1 ${ANSIBLE_PLAYBOOKS}/remote_puppet_afs.yaml
  116. send_timer afs
  117. # Run everything else. We do not care if the other things worked
  118. start_timer
  119. timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/remote_puppet_else.yaml
  120. send_timer else
  121. # Send the combined time for everything
  122. send_timer total $GLOBAL_START_TIME
  123. echo "--- end run @ $(date -Is) ---"
  124. echo