Integrate Elasticsearch and Kibana with Fuel
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

deployment_tasks.yaml 9.0KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288
  1. # Groups definitions
  2. ####################
  3. - id: primary-elasticsearch_kibana
  4. type: group
  5. version: 2.0.0
  6. role: [primary-elasticsearch_kibana]
  7. tasks:
  8. - hiera
  9. - setup_repositories
  10. - fuel_pkgs
  11. - globals
  12. - tools
  13. - logging
  14. - elasticsearch-netconfig
  15. - hosts
  16. - elasticsearch-check-configuration
  17. - elasticsearch-hiera
  18. - elasticsearch-firewall
  19. - elasticsearch-cluster
  20. - elasticsearch-cluster-haproxy
  21. - elasticsearch-virtual-ip
  22. - elasticsearch-haproxy
  23. - elasticsearch-installation
  24. requires: [deploy_start]
  25. required_for: [deploy_end]
  26. parameters:
  27. strategy:
  28. type: one_by_one
  29. - id: elasticsearch_kibana
  30. type: group
  31. version: 2.0.0
  32. role: [elasticsearch_kibana]
  33. tasks:
  34. - hiera
  35. - setup_repositories
  36. - fuel_pkgs
  37. - globals
  38. - tools
  39. - logging
  40. - elasticsearch-netconfig
  41. - hosts
  42. - elasticsearch-check-configuration
  43. - elasticsearch-hiera
  44. - elasticsearch-firewall
  45. - elasticsearch-cluster
  46. - elasticsearch-cluster-haproxy
  47. - elasticsearch-virtual-ip
  48. - elasticsearch-haproxy
  49. - elasticsearch-installation
  50. requires: [deploy_start, primary-elasticsearch_kibana]
  51. required_for: [deploy_end]
  52. parameters:
  53. strategy:
  54. type: parallel
  55. # Tasks definitions for the deployment
  56. ######################################
  57. # This task needs to be reexecuted to adapt the configuration parameters which
  58. # depend on the number of nodes in the cluster
  59. - id: elasticsearch-hiera
  60. type: puppet
  61. version: 2.0.0
  62. requires: [globals]
  63. required_for: [deploy_end]
  64. parameters:
  65. puppet_manifest: "puppet/manifests/hiera_override.pp"
  66. puppet_modules: puppet/modules:/etc/puppet/modules
  67. timeout: 120
  68. reexecute_on:
  69. - deploy_changes
  70. # This task needs to be reexecuted to recheck that the configuration parameters
  71. # match the node's characteristics (eg JVM size).
  72. - id: elasticsearch-check-configuration
  73. type: puppet
  74. version: 2.0.0
  75. requires: [elasticsearch-hiera]
  76. required_for: [deploy_end]
  77. parameters:
  78. puppet_manifest: puppet/manifests/check_environment_configuration.pp
  79. puppet_modules: puppet/modules:/etc/puppet/modules
  80. timeout: 120
  81. reexecute_on:
  82. - deploy_changes
  83. # We cannot use the legacy 'netconfig' task because with MOS 9, it won't create
  84. # the proper dependency on the 'virtual_ips' task for the Elasticsearch-Kibana
  85. # nodes. As a consequence, the deployment will fail because the nodes will
  86. # perform operations that download stuff from the Internet (like running
  87. # 'apt-get update') while the virtual router isn't ready yet. For MOS 8, the
  88. # task depends on the 'tools' task and not on the 'virtual_ips' task otherwise
  89. # it creates a dependency cycle. In this case, this isn't an issue because the
  90. # default gateway is only changed at post-deployment (see the
  91. # 'influxdb-configure-default-route' task).
  92. - id: elasticsearch-netconfig
  93. type: puppet
  94. version: 2.0.0
  95. requires: [elasticsearch-check-configuration, tools]
  96. required_for: [deploy_end, hosts]
  97. cross-depends:
  98. - name: virtual_ips
  99. role: [primary-controller, controller]
  100. parameters:
  101. puppet_manifest: "/etc/puppet/modules/osnailyfacter/modular/netconfig/netconfig.pp"
  102. puppet_modules: /etc/puppet/modules
  103. timeout: 600
  104. reexecute_on:
  105. - deploy_changes
  106. - id: elasticsearch-firewall
  107. type: puppet
  108. version: 2.0.0
  109. requires: [elasticsearch-netconfig]
  110. required_for: [deploy_end]
  111. parameters:
  112. puppet_manifest: "puppet/manifests/firewall.pp"
  113. puppet_modules: puppet/modules:/etc/puppet/modules
  114. timeout: 3600
  115. - id: elasticsearch-cluster
  116. type: puppet
  117. version: 2.0.0
  118. requires: [elasticsearch-firewall]
  119. required_for: [deploy_end]
  120. parameters:
  121. puppet_manifest: "puppet/manifests/cluster.pp"
  122. puppet_modules: puppet/modules:/etc/puppet/modules
  123. timeout: 3600
  124. reexecute_on:
  125. - deploy_changes
  126. - id: elasticsearch-cluster-haproxy
  127. type: puppet
  128. version: 2.0.0
  129. requires: [elasticsearch-cluster]
  130. required_for: [deploy_end]
  131. parameters:
  132. puppet_manifest: "puppet/manifests/cluster-haproxy.pp"
  133. puppet_modules: puppet/modules:/etc/puppet/modules
  134. timeout: 3600
  135. reexecute_on:
  136. - deploy_changes
  137. - id: elasticsearch-virtual-ip
  138. type: puppet
  139. version: 2.0.0
  140. requires: [elasticsearch-cluster]
  141. required_for: [deploy_end]
  142. parameters:
  143. puppet_manifest: "/etc/puppet/modules/osnailyfacter/modular/virtual_ips/virtual_ips.pp"
  144. puppet_modules: /etc/puppet/modules
  145. timeout: 3600
  146. # This task needs to be reexecuted to reconfigure HAProxy backends
  147. - id: elasticsearch-haproxy
  148. type: puppet
  149. version: 2.0.0
  150. requires: [elasticsearch-cluster-haproxy, elasticsearch-virtual-ip]
  151. required_for: [deploy_end]
  152. parameters:
  153. puppet_manifest: "puppet/manifests/haproxy.pp"
  154. puppet_modules: puppet/modules:/etc/puppet/modules
  155. timeout: 120
  156. reexecute_on:
  157. - deploy_changes
  158. # This task needs to be reexecuted to reconfigure Elasticsearch instances
  159. - id: elasticsearch-installation
  160. type: puppet
  161. version: 2.0.0
  162. requires: [elasticsearch-haproxy]
  163. required_for: [deploy_end]
  164. cross-depends:
  165. - name: telemetry-hiera
  166. parameters:
  167. puppet_manifest: puppet/manifests/elasticsearch.pp
  168. puppet_modules: puppet/modules:/etc/puppet/modules
  169. timeout: 600
  170. reexecute_on:
  171. - deploy_changes
  172. # Tasks defintions for the post-deployment
  173. ##########################################
  174. - id: elasticsearch-dns-client
  175. type: puppet
  176. version: 2.0.0
  177. role: [primary-elasticsearch_kibana, elasticsearch_kibana]
  178. requires: [post_deployment_start]
  179. required_for: [post_deployment_end]
  180. parameters:
  181. puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/dns/dns-client.pp
  182. puppet_modules: /etc/puppet/modules
  183. timeout: 600
  184. - id: elasticsearch-ntp-client
  185. type: puppet
  186. version: 2.0.0
  187. role: [primary-elasticsearch_kibana, elasticsearch_kibana]
  188. requires: [elasticsearch-dns-client]
  189. required_for: [post_deployment_end]
  190. parameters:
  191. puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/ntp/ntp-client.pp
  192. puppet_modules: /etc/puppet/modules
  193. timeout: 600
  194. # Configure the default gateway (required for MOS 8 only)
  195. - id: elasticsearch-configure-default-route
  196. type: puppet
  197. version: 2.0.0
  198. role: [primary-elasticsearch_kibana, elasticsearch_kibana]
  199. requires: [post_deployment_start]
  200. required_for: [post_deployment_end]
  201. parameters:
  202. puppet_manifest: "puppet/manifests/configure_default_route.pp"
  203. puppet_modules: /etc/puppet/modules
  204. timeout: 120
  205. reexecute_on:
  206. - deploy_changes
  207. # In order to automatically import dashboards, it is mandatory to create and
  208. # configure the Kibana index before Kibana4 starts. This task is ran at post-deployment
  209. # stage because the index creation must occur when the Elasticsearch cluster is ready
  210. # (eg. all instances are started). This is required for MOS 8 (role-based deployment).
  211. - id: provision-kibana-index
  212. type: puppet
  213. version: 2.0.0
  214. role: [primary-elasticsearch_kibana]
  215. requires: [post_deployment_start]
  216. required_for: [post_deployment_end]
  217. parameters:
  218. puppet_manifest: puppet/manifests/kibana_index_configuration.pp
  219. puppet_modules: puppet/modules:/etc/puppet/modules
  220. timeout: 600
  221. - id: kibana-installation
  222. type: puppet
  223. version: 2.0.0
  224. role: [primary-elasticsearch_kibana, elasticsearch_kibana]
  225. cross-depends:
  226. - name: provision-kibana-index
  227. role: [primary-elasticsearch_kibana]
  228. requires: [post_deployment_start, provision-kibana-index]
  229. required_for: [post_deployment_end]
  230. parameters:
  231. puppet_manifest: puppet/manifests/kibana.pp
  232. puppet_modules: puppet/modules:/etc/puppet/modules
  233. timeout: 600
  234. # This task needs to be reexecuted to reconfigure the index templates with an
  235. # appropriate number_of_replicas.
  236. # This is done at post_deployment time to ensure that the cluster is ready.
  237. - id: elasticsearch-kibana-configuration
  238. type: puppet
  239. version: 2.0.0
  240. role: [primary-elasticsearch_kibana]
  241. requires: [post_deployment_start, kibana-installation]
  242. # We use upload_nodes_info as an anchor to order the post-deployment tasks executed
  243. # by this plugin and the LMA collector plugin. The dependency chain is like this:
  244. # elasticsearch-kibana-configuration -> upload_nodes_info -> (LMA collector tasks)
  245. required_for: [upload_nodes_info, post_deployment_end]
  246. cross-depended-by:
  247. - name: lma-configure-apt
  248. role: /.*/
  249. parameters:
  250. puppet_manifest: puppet/manifests/provision_services.pp
  251. puppet_modules: puppet/modules:/etc/puppet/modules
  252. timeout: 600
  253. reexecute_on:
  254. - deploy_changes
  255. # This task needs to be reexecuted to reconfigure the no-quorum-policy every
  256. # time the cluster is scaled up or down. In case of scale up, it should happen
  257. # after the new nodes have added themselves to the Pacemaker cluster.
  258. - id: elasticsearch-enable-quorum
  259. type: puppet
  260. version: 2.0.0
  261. role: [primary-elasticsearch_kibana]
  262. requires: [elasticsearch-kibana-configuration]
  263. required_for: [post_deployment_end]
  264. parameters:
  265. puppet_manifest: "puppet/manifests/enable_quorum.pp"
  266. puppet_modules: puppet/modules:/etc/puppet/modules
  267. timeout: 120
  268. reexecute_on:
  269. - deploy_changes