e08c58dd15
This change makes the retention gathering operation faster by storing the retention values as "local facts". The local facts are then referenced in templates loading from the local fact file instead of running repetitive queries which are slow making very large deployments cumbersome. To make the retention policy fact gathering process smarter it will now automatically refresh if undefined or should the elasticsearch cluster size change. This will ensure we're improving speed of execution while also catering to the needs of deployers to grow, or shrink, elasticsearch cluster sizes. Documentation has been added regarding the new option and why it may be of use to depoyers. Change-Id: I3936ee94461ac39fb8bc78dc2c873e6067552461 Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
103 lines
3.1 KiB
YAML
103 lines
3.1 KiB
YAML
---
|
|
# Copyright 2018, Rackspace US, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
- name: Ensure local facts directory exists
|
|
file:
|
|
dest: "/etc/ansible/facts.d"
|
|
state: directory
|
|
group: "root"
|
|
owner: "root"
|
|
mode: "0755"
|
|
recurse: no
|
|
|
|
- name: Initialize local facts
|
|
ini_file:
|
|
dest: "/etc/ansible/facts.d/elastic.fact"
|
|
section: "retention"
|
|
option: cacheable
|
|
value: true
|
|
|
|
- name: Refresh local facts
|
|
setup:
|
|
filter: ansible_local
|
|
gather_subset: "!all"
|
|
tags:
|
|
- always
|
|
|
|
- name: Retention storage block
|
|
block:
|
|
- name: Query es storage
|
|
uri:
|
|
url: "http://{{ coordination_nodes[0] }}/_nodes/{{ (data_nodes | map('extract', hostvars, 'ansible_host') | list) | join(',') }}/stats/fs"
|
|
method: GET
|
|
register: elk_data
|
|
environment:
|
|
no_proxy: "{{ coordination_nodes[0].split(':')[0] }}"
|
|
until:
|
|
- elk_data is success and elk_data['json'] is defined
|
|
retries: 5
|
|
delay: 30
|
|
run_once: true
|
|
|
|
- name: Set retention keys fact
|
|
set_fact:
|
|
es_storage_json: "{{ elk_data['json'] }}"
|
|
|
|
- name: Load retention algo variables
|
|
include_vars: "calculate_index_retention_{{ elastic_index_retention_algorithm }}.yml"
|
|
tags:
|
|
- always
|
|
|
|
- name: Set storage fact
|
|
ini_file:
|
|
dest: "/etc/ansible/facts.d/elastic.fact"
|
|
section: "retention"
|
|
option: "cluster_nodes"
|
|
value: "{{ groups['elastic-logstash'] | length }}"
|
|
|
|
- name: Set retention policy fact
|
|
ini_file:
|
|
dest: "/etc/ansible/facts.d/elastic.fact"
|
|
section: "retention"
|
|
option: "elastic_beat_retention_policy_keys"
|
|
value: "{{ elastic_beat_retention_policy_hosts.keys() | list }}"
|
|
|
|
- name: Set size fact
|
|
ini_file:
|
|
dest: "/etc/ansible/facts.d/elastic.fact"
|
|
section: "retention"
|
|
option: "elastic_{{ item.key }}_size"
|
|
value: "{{ item.value }}"
|
|
with_dict: "{{ es_storage_per_index }}"
|
|
|
|
- name: Set retention fact
|
|
ini_file:
|
|
dest: "/etc/ansible/facts.d/elastic.fact"
|
|
section: "retention"
|
|
option: "elastic_{{ item.key }}_retention"
|
|
value: "{{ item.value }}"
|
|
with_dict: "{{ es_days_per_index }}"
|
|
|
|
- name: Refresh local facts
|
|
setup:
|
|
filter: ansible_local
|
|
gather_subset: "!all"
|
|
tags:
|
|
- always
|
|
when:
|
|
- (ansible_local['elastic']['retention']['cluster_nodes'] is undefined) or
|
|
((groups['elastic-logstash'] | length) != (ansible_local['elastic']['retention']['cluster_nodes'] | int)) or
|
|
(elastic_retention_refresh | bool)
|