openstack/openstack-ansible-ops
Code Issues Proposed changes

Merge "Add the ability to enable or disable rollups / indexes"

Browse Source
This commit is contained in:
Zuul 2019-02-12 06:33:05 +00:00 committed by Gerrit Code Review
commit e30f3f0391
12 changed files with 160 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

141
elk_metrics_6x/createElasticIndexes.yml
View File

@ -28,6 +28,43 @@
- role: elastic_retention
post_tasks:
- name: Create beat indexes
uri:
url: http://127.0.0.1:9200/{{ item.name }}
method: PUT
body: "{{ item.index_options | to_json }}"
status_code: 200,400
body_format: json
register: elk_indexes
until: elk_indexes is success
retries: 3
delay: 30
with_items: |-
{% set beat_indexes = [] %}
{% for key, value in elastic_beat_retention_policy_hosts.items() %}
{% if ((value.hosts | length) > 0) and (value.make_index | default(false) | bool) %}
{%
set _index = {
'name': key,
'index_options': {
'settings': {
'index': {
'codec': 'best_compression',
'mapping': {
'total_fields': {
'limit': '10000'
}
},
'refresh_interval': elastic_refresh_interval
}
}
}
}
%}
{% set _ = beat_indexes.append(_index) %}
{% endif %}
{% endfor %}
{{ beat_indexes }}
- name: Create basic indexes
uri:
url: http://127.0.0.1:9200/{{ item.name }}
@ -40,15 +77,6 @@
retries: 3
delay: 30
with_items:
- name: "osprofiler-notifications"
index_options:
settings:
index:
codec: "best_compression"
mapping:
total_fields:
limit: "10000"
refresh_interval: "{{ elastic_refresh_interval }}"
- name: "_all/_settings?preserve_existing=true"
index_options:
index.queries.cache.enabled: "true"
@ -120,8 +148,101 @@
delay: 30
vars:
index_option:
template: ".monitoring-*"
template: ".monitoring*"
order: 1
settings:
number_of_replicas: "{{ elasticsearch_number_of_replicas | int }}"
number_of_shards: "{{ ((elasticsearch_number_of_replicas | int) * 2) + 1 }}"
- name: Create custom skydive index template
uri:
url: http://127.0.0.1:9200/_template/skydive
method: PUT
body: "{{ index_option | to_json }}"
status_code: 200
body_format: json
register: create_basicIndexTemplate
until: create_basicIndexTemplate is success
retries: 3
delay: 30
vars:
index_option:
template: "skydive*"
order: 1
settings:
number_of_replicas: "{{ elasticsearch_number_of_replicas | int }}"
number_of_shards: "{{ ((elasticsearch_number_of_replicas | int) * 2) + 1 }}"
- name: Create/Setup known indexes in Kibana
hosts: kibana
become: true
vars_files:
- vars/variables.yml
environment: "{{ deployment_environment_variables | default({}) }}"
roles:
- role: elastic_retention
post_tasks:
- name: Create kibana indexe patterns
uri:
url: "http://127.0.0.1:5601/api/saved_objects/index-pattern/{{ item.name }}"
method: POST
body: "{{ item.index_options | to_json }}"
status_code: 200,409
body_format: json
headers:
Content-Type: "application/json"
kbn-xsrf: "{{ inventory_hostname | to_uuid }}"
with_items: |-
{% set beat_indexes = [] %}
{% for key, value in elastic_beat_retention_policy_hosts.items() %}
{% if (value.hosts | length) > 0 %}
{%
set _index = {
'name': key,
'index_options': {
'attributes': {
'title': (key ~ '*')
}
}
}
%}
{% if value.timeFieldName is defined %}
{% set _ = _index.index_options.attributes.__setitem__('timeFieldName', (value.timeFieldName | string)) %}
{% endif %}
{% set _ = beat_indexes.append(_index) %}
{% endif %}
{% endfor %}
{% set _ = beat_indexes.append({'name': 'default', 'index_options': {'attributes': {'title': '*'}}}) %}
{{ beat_indexes }}
register: kibana_indexes
until: kibana_indexes is success
retries: 6
delay: 30
run_once: true
- name: Create basic indexes
uri:
url: "http://127.0.0.1:5601/api/kibana/settings/defaultIndex"
method: POST
body: "{{ item.index_options | to_json }}"
status_code: 200
body_format: json
headers:
Content-Type: "application/json"
kbn-xsrf: "{{ inventory_hostname | to_uuid }}"
with_items:
- name: "default"
index_options:
value: "default"
register: kibana_indexes
until: kibana_indexes is success
retries: 6
delay: 30
run_once: true
tags:
- server-install

6
elk_metrics_6x/installAPMserver.yml
View File

@ -26,7 +26,8 @@
- role: elastic_apm_server
- role: elastic_rollup
index_name: apm
when:
- elastic_create_rollup | bool
tags:
- apm-server
@ -44,6 +45,7 @@
roles:
- role: elastic_rollup
index_name: apm
when:
- elastic_create_rollup | bool
tags:
- apm-server

3
elk_metrics_6x/installAuditbeat.yml
View File

@ -42,7 +42,8 @@
roles:
- role: elastic_rollup
index_name: auditbeat
when:
- elastic_create_rollup | bool
tags:
- auditbeat

3
elk_metrics_6x/installFilebeat.yml
View File

@ -42,7 +42,8 @@
roles:
- role: elastic_rollup
index_name: filebeat
when:
- elastic_create_rollup | bool
tags:
- filebeat

3
elk_metrics_6x/installHeartbeat.yml
View File

@ -56,7 +56,8 @@
roles:
- role: elastic_rollup
index_name: heartbeat
when:
- elastic_create_rollup | bool
tags:
- heartbeat

3
elk_metrics_6x/installJournalbeat.yml
View File

@ -96,6 +96,7 @@
roles:
- role: elastic_rollup
index_name: journalbeat
when:
- elastic_create_rollup | bool
tags:
- journalbeat

43
elk_metrics_6x/installKibana.yml
View File

@ -22,48 +22,5 @@
roles:
- role: elastic_kibana
post_tasks:
- name: Create basic indexes
uri:
url: "http://127.0.0.1:5601/api/saved_objects/index-pattern/{{ item.name }}"
method: POST
body: "{{ item.index_options | to_json }}"
status_code: 200,409
body_format: json
headers:
Content-Type: "application/json"
kbn-xsrf: "{{ inventory_hostname | to_uuid }}"
with_items:
- name: "*"
index_options:
attributes:
title: "*"
timeFieldName: "@timestamp"
register: kibana_indexes
until: kibana_indexes is success
retries: 6
delay: 30
run_once: true
- name: Create basic indexes
uri:
url: "http://127.0.0.1:5601/api/kibana/settings/defaultIndex"
method: POST
body: "{{ item.index_options | to_json }}"
status_code: 200
body_format: json
headers:
Content-Type: "application/json"
kbn-xsrf: "{{ inventory_hostname | to_uuid }}"
with_items:
- name: "*"
index_options:
value: "*"
register: kibana_indexes
until: kibana_indexes is success
retries: 6
delay: 30
run_once: true
tags:
- server-install

3
elk_metrics_6x/installMetricbeat.yml
View File

@ -42,7 +42,8 @@
roles:
- role: elastic_rollup
index_name: metricbeat
when:
- elastic_create_rollup | bool
tags:
- metricbeat

3
elk_metrics_6x/installPacketbeat.yml
View File

@ -42,7 +42,8 @@
roles:
- role: elastic_rollup
index_name: packetbeat
when:
- elastic_create_rollup | bool
tags:
- packetbeat

1
elk_metrics_6x/roles/elastic_journalbeat/tasks/systemd.general-overrides.conf.j2
View File

@ -1 +0,0 @@
../../../templates/systemd.general-overrides.conf.j2

14
elk_metrics_6x/roles/elastic_retention/defaults/main.yml
View File

@ -70,35 +70,45 @@ elastic_index_retention_algorithm: default
elastic_beat_retention_policy_hosts:
logstash:
make_index: true
weight: 1
hosts: "{{ groups['elastic-logstash'] | default([]) }}"
apm:
make_index: true
timeFieldName: '@timestamp'
weight: 1
hosts: "{{ groups['apm-server'] | default([]) }}"
auditbeat:
timeFieldName: '@timestamp'
weight: 10
hosts: "{{ groups['hosts'] | default([]) }}"
filebeat:
timeFieldName: '@timestamp'
weight: 10
hosts: "{{ groups['hosts'] | default([]) }}"
syslog:
make_index: true
weight: 1
hosts: "{{ groups['hosts'] | default([]) }}"
heartbeat:
timeFieldName: '@timestamp'
weight: 1
hosts: "{{ groups['kibana'][:3] | default([]) }}"
journalbeat:
timeFieldName: '@timestamp'
weight: 3
hosts: "{{ groups['all'] | default([]) }}"
hosts: "{{ groups['hosts'] | default([]) }}"
metricbeat:
timeFieldName: '@timestamp'
weight: 2
hosts: "{{ groups['all'] | default([]) }}"
packetbeat:
timeFieldName: '@timestamp'
weight: 1
hosts: "{{ groups['hosts'] | default([]) }}"
skydive:
weight: 1
hosts: "{{ groups['skydive_analyzers'] | default([]) }}"
hosts: "{{ (((groups['skydive_analyzers'] | default([])) | length) > 0) | ternary((groups['hosts'] | default([])), []) }}"
# Refresh the elasticsearch retention policy local facts.
elastic_retention_refresh: false

1
elk_metrics_6x/vars/variables.yml
View File

@ -11,6 +11,7 @@ q_storage: "{{ (ansible_processor_count | int) * (ansible_processor_threads_per_
apm_port: 8200
elastic_port: 9200
elastic_hap_port: 9201
elastic_create_rollup: false
logstash_beat_input_port: 5044
logstash_syslog_input_port: 5140
logstash_syslog_input_mode: udp