b6343c57a4
The logstash groks were running in line using the legacy method which uses lexical sorting of all logstash filter files and loads them in order. While this works it makes it so all data has to travel through all filters. This change makes use of the logstash multi-pipeline capabilities using a distributor and fork pattern. This allows data to flow through logstash more quickly and not block whenever there's an issue with an output plugin. Finger-prints using SHA1 when there's a message and UUID when not. This will ensure we're duplicating log entries which will help speed up transations and further reduce the storage required. Change-Id: I38268e33b370da0f1e186ecf65911d4a312c3e6a Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
125 lines
3.4 KiB
Django/Jinja
125 lines
3.4 KiB
Django/Jinja
---
|
|
# Copyright 2018, Rackspace US, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
{% set action_items = [] -%}
|
|
{# Delete index loop #}
|
|
{% for key in elastic_beat_retention_policy_hosts.keys() -%}
|
|
{% set delete_indices = {} -%}
|
|
{% set index_retention = hostvars[inventory_hostname]['elastic_' + key + '_retention'] -%}
|
|
{% set _ = delete_indices.update(
|
|
{
|
|
'action': 'delete_indices',
|
|
'description': 'Prune indices for ' + key + ' after ' ~ (index_retention | int) ~ ' days.',
|
|
'options': {
|
|
'ignore_empty_list': true,
|
|
'disable_action': false
|
|
}
|
|
}
|
|
)
|
|
-%}
|
|
{# add the filter loop #}
|
|
{% set filters = [] -%}
|
|
{% set _ = filters.append(
|
|
{
|
|
'filtertype': 'pattern',
|
|
'kind': 'prefix',
|
|
'value': key + '-'
|
|
}
|
|
)
|
|
-%}
|
|
{% set _ = filters.append(
|
|
{
|
|
'filtertype': 'age',
|
|
'source': 'name',
|
|
'direction': 'older',
|
|
'timestring': '%Y.%m.%d',
|
|
'unit': 'days',
|
|
'unit_count': (index_retention | int)
|
|
}
|
|
)
|
|
-%}
|
|
{% set _ = delete_indices.update({'filters': filters}) -%}
|
|
{% set _ = action_items.append(delete_indices) -%}
|
|
|
|
{# Set shrink curator options #}
|
|
{% set shrink_indices = {} -%}
|
|
{% set _ = shrink_indices.update(
|
|
{
|
|
'action': 'shrink',
|
|
'description': 'Shrink ' + key + ' indices older than ' ~ (index_retention | int) // 4 ~ ' days',
|
|
'options': {
|
|
"disable_action": false,
|
|
"ignore_empty_list": true,
|
|
"shrink_node": "DETERMINISTIC",
|
|
"node_filters": {
|
|
"permit_masters": ((master_nodes | length) < (data_nodes | length)) | ternary(true, false),
|
|
"exclude_nodes": (groups['kibana'] | map('extract', hostvars, 'ansible_host') | list)
|
|
},
|
|
"number_of_shards": 1,
|
|
"number_of_replicas": elasticsearch_number_of_replicas,
|
|
"shrink_suffix": '-shrink',
|
|
"copy_aliases": true,
|
|
"delete_after": true,
|
|
"post_allocation": {
|
|
"allocation_type": "include",
|
|
"key": "node_tag",
|
|
"value": "cold"
|
|
},
|
|
"wait_for_active_shards": 1,
|
|
"extra_settings": {
|
|
"settings": {
|
|
"index.codec": "best_compression"
|
|
}
|
|
},
|
|
"wait_for_completion": true,
|
|
"wait_for_rebalance": true,
|
|
"wait_interval": 9,
|
|
"max_wait": -1
|
|
}
|
|
}
|
|
)
|
|
-%}
|
|
{% set filters = [] -%}
|
|
{% set _ = filters.append(
|
|
{
|
|
'filtertype': 'pattern',
|
|
'kind': 'prefix',
|
|
'value': key + '-'
|
|
}
|
|
)
|
|
-%}
|
|
{% set _ = filters.append(
|
|
{
|
|
'filtertype': 'age',
|
|
'source': 'creation_date',
|
|
'direction': 'older',
|
|
'unit': 'days',
|
|
'unit_count': (index_retention | int) // 4
|
|
}
|
|
)
|
|
-%}
|
|
{% set _ = shrink_indices.update({'filters': filters}) -%}
|
|
{% set _ = action_items.append(shrink_indices) -%}
|
|
{% endfor -%}
|
|
|
|
{% set actions = {} -%}
|
|
{% for action_item in action_items -%}
|
|
{% set _ = actions.update({loop.index: action_item}) -%}
|
|
{% endfor -%}
|
|
|
|
{# Render all actions #}
|
|
{% set curator_actions = {'actions': actions} -%}
|
|
{{ curator_actions | to_nice_yaml(indent=2) }}
|