From d4381456fda110a7cc2faa0d47b5bc0ae1352abc Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Wed, 13 Nov 2019 13:53:48 +0000 Subject: [PATCH] Support deploying Elasticsearch Curator This allows users to supply an Elasticsearch Curator actions file to manage log retention [1]. Curator then runs on a cron job, which defaults to every day. A default curator actions file is provided, which can be customised by the end user if required. [1] https://www.elastic.co/guide/en/elasticsearch/client/curator/current/actionfile.html Change-Id: Ide9baea9190ae849e61b9d8b6cff3305bdcdd534 --- ansible/group_vars/all.yml | 5 +++ ansible/inventory/all-in-one | 4 ++ ansible/inventory/multinode | 4 ++ ansible/roles/elasticsearch/defaults/main.yml | 45 +++++++++++++++++++ ansible/roles/elasticsearch/handlers/main.yml | 15 +++++++ .../elasticsearch/tasks/check-containers.yml | 2 +- ansible/roles/elasticsearch/tasks/config.yml | 35 ++++++++++++++- ansible/roles/elasticsearch/tasks/pull.yml | 2 +- .../elasticsearch-curator-actions.yml.j2 | 33 ++++++++++++++ .../elasticsearch-curator.crontab.j2 | 1 + .../templates/elasticsearch-curator.json.j2 | 31 +++++++++++++ .../templates/elasticsearch-curator.yml.j2 | 8 ++++ .../central-logging-guide.rst | 34 ++++++++++++++ etc/kolla/globals.yml | 1 + ...lasticsearch-curator-88089d04f7ccd549.yaml | 4 ++ 15 files changed, 220 insertions(+), 4 deletions(-) create mode 100644 ansible/roles/elasticsearch/templates/elasticsearch-curator-actions.yml.j2 create mode 100644 ansible/roles/elasticsearch/templates/elasticsearch-curator.crontab.j2 create mode 100644 ansible/roles/elasticsearch/templates/elasticsearch-curator.json.j2 create mode 100644 ansible/roles/elasticsearch/templates/elasticsearch-curator.yml.j2 create mode 100644 releasenotes/notes/add-elasticsearch-curator-88089d04f7ccd549.yaml diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 000e53df37..b5c77770b4 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -724,6 +724,11 @@ skip_stop_containers: [] elasticsearch_address: "{{ kolla_internal_fqdn }}" enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_osprofiler | bool or enable_skydive | bool or enable_monasca | bool else 'no' }}" + +# If using Curator an actions file will need to be defined. Please see +# the documentation. +enable_elasticsearch_curator: "no" + enable_kibana: "{{ 'yes' if enable_central_logging | bool or enable_monasca | bool else 'no' }}" #################### diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index 50ffb89d9e..0808b29c22 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -259,6 +259,10 @@ control # function appropriately. For example, neutron-metadata-agent must run on the # same host as the l3-agent and (depending on configuration) the dhcp-agent. +# Elasticsearch Curator +[elasticsearch-curator:children] +elasticsearch + # Glance [glance-api:children] glance diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index 94f3af02fb..03d0619429 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -278,6 +278,10 @@ control # function appropriately. For example, neutron-metadata-agent must run on the # same host as the l3-agent and (depending on configuration) the dhcp-agent. +# Elasticsearch Curator +[elasticsearch-curator:children] +elasticsearch + # Glance [glance-api:children] glance diff --git a/ansible/roles/elasticsearch/defaults/main.yml b/ansible/roles/elasticsearch/defaults/main.yml index 5aabd95ca1..4d57b85c0d 100644 --- a/ansible/roles/elasticsearch/defaults/main.yml +++ b/ansible/roles/elasticsearch/defaults/main.yml @@ -19,6 +19,13 @@ elasticsearch_services: port: "{{ elasticsearch_port }}" frontend_http_extra: - "option dontlog-normal" + elasticsearch-curator: + container_name: elasticsearch_curator + group: elasticsearch-curator + enabled: "{{ enable_elasticsearch_curator }}" + image: "{{ elasticsearch_curator_image_full }}" + volumes: "{{ elasticsearch_curator_default_volumes + elasticsearch_curator_extra_volumes }}" + dimensions: "{{ elasticsearch_curator_dimensions }}" #################### @@ -28,6 +35,33 @@ elasticsearch_cluster_name: "kolla_logging" es_heap_size: "1g" es_java_opts: "{% if es_heap_size %}-Xms{{ es_heap_size }} -Xmx{{ es_heap_size }}{%endif%}" +####################### +# Elasticsearch Curator +####################### + +# How frequently Curator runs. On multinode deployments of Curator +# you may wish to override this in hostvars so that Curator does +# not run simultaneously on all nodes. Defaults to every midnight. +elasticsearch_curator_cron_schedule: "0 0 * * *" + +# When set to True, Curator will not modify Elasticsearch data, but +# will print what it *would* do to the Curator log file. This is a +# useful way of checking that Curator actions are working as expected. +elasticsearch_curator_dry_run: false + +# Index prefix pattern. Any indices matching this regex will +# be managed by Curator. +elasticsearch_curator_index_pattern: "^{{ 'monasca' if enable_monasca|bool else kibana_log_prefix }}-.*" + +# Duration after which an index is staged for deletion. This is +# implemented by closing the index. Whilst in this state the index +# contributes negligible load on the cluster and may be manually +# re-opened if required. +elasticsearch_curator_soft_retention_period_days: 30 + +# Duration after which an index is permanently erased from the cluster. +elasticsearch_curator_hard_retention_period_days: 60 + #################### # Docker #################### @@ -36,10 +70,21 @@ elasticsearch_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ d elasticsearch_tag: "{{ openstack_release }}" elasticsearch_image_full: "{{ elasticsearch_image }}:{{ elasticsearch_tag }}" +elasticsearch_curator_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ elasticsearch_install_type }}-elasticsearch-curator" +elasticsearch_curator_tag: "{{ openstack_release }}" +elasticsearch_curator_image_full: "{{ elasticsearch_curator_image }}:{{ elasticsearch_curator_tag }}" + elasticsearch_dimensions: "{{ default_container_dimensions }}" +elasticsearch_curator_dimensions: "{{ default_container_dimensions }}" elasticsearch_default_volumes: - "{{ node_config_directory }}/elasticsearch/:{{ container_config_directory }}/" - "/etc/localtime:/etc/localtime:ro" - "{{ elasticsearch_datadir_volume }}:/var/lib/elasticsearch/data" +elasticsearch_curator_default_volumes: + - "{{ node_config_directory }}/elasticsearch-curator/:{{ container_config_directory }}/" + - "/etc/localtime:/etc/localtime:ro" + - "kolla_logs:/var/log/kolla" + elasticsearch_extra_volumes: "{{ default_extra_volumes }}" +elasticsearch_curator_extra_volumes: "{{ default_extra_volumes }}" diff --git a/ansible/roles/elasticsearch/handlers/main.yml b/ansible/roles/elasticsearch/handlers/main.yml index 790cfb7d9f..af1cf3282d 100644 --- a/ansible/roles/elasticsearch/handlers/main.yml +++ b/ansible/roles/elasticsearch/handlers/main.yml @@ -14,3 +14,18 @@ dimensions: "{{ service.dimensions }}" when: - kolla_action != "config" + +- name: Restart elasticsearch-curator container + vars: + service_name: "elasticsearch-curator" + service: "{{ elasticsearch_services[service_name] }}" + become: true + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + dimensions: "{{ service.dimensions }}" + when: + - kolla_action != "config" diff --git a/ansible/roles/elasticsearch/tasks/check-containers.yml b/ansible/roles/elasticsearch/tasks/check-containers.yml index b48b95fef0..b6ebbbb2be 100644 --- a/ansible/roles/elasticsearch/tasks/check-containers.yml +++ b/ansible/roles/elasticsearch/tasks/check-containers.yml @@ -8,7 +8,7 @@ image: "{{ item.value.image }}" volumes: "{{ item.value.volumes }}" dimensions: "{{ item.value.dimensions }}" - environment: "{{ item.value.environment }}" + environment: "{{ item.value.environment|default(omit) }}" when: - inventory_hostname in groups[item.value.group] - item.value.enabled | bool diff --git a/ansible/roles/elasticsearch/tasks/config.yml b/ansible/roles/elasticsearch/tasks/config.yml index 0e0e626e7b..c8fbc75eeb 100644 --- a/ansible/roles/elasticsearch/tasks/config.yml +++ b/ansible/roles/elasticsearch/tasks/config.yml @@ -34,9 +34,9 @@ notify: - Restart {{ item.key }} container -- name: Copying over elasticsearch.yml +- name: Copying over elasticsearch service config files template: - src: "elasticsearch.yml.j2" + src: "{{ item.key }}.yml.j2" dest: "{{ node_config_directory }}/{{ item.key }}/{{ item.key }}.yml" mode: "0660" become: true @@ -47,5 +47,36 @@ notify: - Restart {{ item.key }} container +- name: Copying over elasticsearch curator actions + vars: + service: "{{ elasticsearch_services['elasticsearch-curator'] }}" + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/elasticsearch-curator/elasticsearch-curator-actions.yml" + mode: "0660" + become: true + when: + - inventory_hostname in groups[service['group']] + - service.enabled | bool + with_first_found: + - "{{ node_custom_config }}/elasticsearch/elasticsearch-curator-actions.yml" + - "{{ role_path }}/templates/elasticsearch-curator-actions.yml.j2" + notify: + - Restart elasticsearch-curator container + +- name: Copying over elasticsearch curator crontab + vars: + service: "{{ elasticsearch_services['elasticsearch-curator'] }}" + template: + src: "{{ role_path }}/templates/elasticsearch-curator.crontab.j2" + dest: "{{ node_config_directory }}/elasticsearch-curator/elasticsearch-curator.crontab" + mode: "0660" + become: true + when: + - inventory_hostname in groups[service['group']] + - service.enabled | bool + notify: + - Restart elasticsearch-curator container + - include_tasks: check-containers.yml when: kolla_action != "config" diff --git a/ansible/roles/elasticsearch/tasks/pull.yml b/ansible/roles/elasticsearch/tasks/pull.yml index bd2ab6ba45..02435c5f59 100644 --- a/ansible/roles/elasticsearch/tasks/pull.yml +++ b/ansible/roles/elasticsearch/tasks/pull.yml @@ -1,5 +1,5 @@ --- -- name: Pulling elasticsearch image +- name: Pulling elasticsearch images become: true kolla_docker: action: "pull_image" diff --git a/ansible/roles/elasticsearch/templates/elasticsearch-curator-actions.yml.j2 b/ansible/roles/elasticsearch/templates/elasticsearch-curator-actions.yml.j2 new file mode 100644 index 0000000000..0ef452874b --- /dev/null +++ b/ansible/roles/elasticsearch/templates/elasticsearch-curator-actions.yml.j2 @@ -0,0 +1,33 @@ +actions: + 1: + action: close + description: >- + Closes indices + options: + ignore_empty_list: True + filters: + - filtertype: pattern + kind: prefix + value: "{{ elasticsearch_curator_index_pattern }}" + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: "{{ elasticsearch_curator_soft_retention_period_days }}" + 2: + action: delete_indices + description: >- + Delete indicies + options: + ignore_empty_list: True + filters: + - filtertype: pattern + kind: prefix + value: "{{ elasticsearch_curator_index_pattern }}" + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: "{{ elasticsearch_curator_hard_retention_period_days }}" diff --git a/ansible/roles/elasticsearch/templates/elasticsearch-curator.crontab.j2 b/ansible/roles/elasticsearch/templates/elasticsearch-curator.crontab.j2 new file mode 100644 index 0000000000..25bf882c68 --- /dev/null +++ b/ansible/roles/elasticsearch/templates/elasticsearch-curator.crontab.j2 @@ -0,0 +1 @@ +{{ elasticsearch_curator_cron_schedule }} curator --config /etc/elasticsearch-curator/curator.yml {% if elasticsearch_curator_dry_run|bool %}--dry-run {% endif %}/etc/elasticsearch-curator/actions.yml diff --git a/ansible/roles/elasticsearch/templates/elasticsearch-curator.json.j2 b/ansible/roles/elasticsearch/templates/elasticsearch-curator.json.j2 new file mode 100644 index 0000000000..579c98387e --- /dev/null +++ b/ansible/roles/elasticsearch/templates/elasticsearch-curator.json.j2 @@ -0,0 +1,31 @@ +{% set cron_cmd = 'cron -f' if kolla_base_distro in ['ubuntu', 'debian'] else 'crond -s -n' %} +{ + "command": "{{ cron_cmd }}", + "config_files": [ + { + "source": "{{ container_config_directory }}/elasticsearch-curator.crontab", + "dest": "/var/spool/cron/elasticsearch", + "owner": "root", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/elasticsearch-curator.yml", + "dest": "/etc/elasticsearch-curator/curator.yml", + "owner": "elasticsearch", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/elasticsearch-curator-actions.yml", + "dest": "/etc/elasticsearch-curator/actions.yml", + "owner": "elasticsearch", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/elasticsearch", + "owner": "elasticsearch:elasticsearch", + "recurse": true + } + ] +} diff --git a/ansible/roles/elasticsearch/templates/elasticsearch-curator.yml.j2 b/ansible/roles/elasticsearch/templates/elasticsearch-curator.yml.j2 new file mode 100644 index 0000000000..544f554e8d --- /dev/null +++ b/ansible/roles/elasticsearch/templates/elasticsearch-curator.yml.j2 @@ -0,0 +1,8 @@ +client: + hosts: [{% for host in groups['elasticsearch'] %}"{{ 'api' | kolla_address(host) }}"{% if not loop.last %},{% endif %}{% endfor %}] + port: {{ elasticsearch_port }} + timeout: 30 + +logging: + loglevel: INFO + logfile: /var/log/kolla/elasticsearch/elasticsearch-curator.log diff --git a/doc/source/reference/logging-and-monitoring/central-logging-guide.rst b/doc/source/reference/logging-and-monitoring/central-logging-guide.rst index e5058514a4..1376950ba3 100644 --- a/doc/source/reference/logging-and-monitoring/central-logging-guide.rst +++ b/doc/source/reference/logging-and-monitoring/central-logging-guide.rst @@ -35,6 +35,40 @@ By default Elasticsearch is deployed on port ``9200``. ``elasticsearch`` to store the data of Elasticsearch. The path can be set via the variable ``elasticsearch_datadir_volume``. +Curator +------- + +To stop your disks filling up, retention policies can be set. These are +enforced by Elasticsearch Curator which can be enabled by setting the +following in ``/etc/kolla/globals.yml``: + +.. code-block:: yaml + + enable_elasticsearch_curator: "yes" + +Elasticsearch Curator is configured via an actions file. The format of the +actions file is described in the `Elasticsearch Curator documentation `_. +A default actions file is provided which closes indices and then deletes them +some time later. The periods for these operations, as well as the prefix for +determining which indicies should be managed are defined in the Elasticsearch +role defaults and can be overridden in ``/etc/kolla/globals.yml`` if required. + +If the default actions file is not malleable enough, a custom actions file can +be placed in the Kolla custom config directory, for example: +``/etc/kolla/config/elasticsearch/elasticsearch-curator-actions.yml``. + +When testing the actions file you may wish to perform a dry run to be certain +of what Curator will actually do. A dry run can be enabled by setting the +following in ``/etc/kolla/globals.yml``: + +.. code-block:: yaml + + elasticsearch_curator_dry_run: "yes" + +The actions which *would* be taken if a dry run were to be disabled are then +logged in the Elasticsearch Kolla logs folder under +``/var/log/kolla/elasticsearch/elasticsearch-curator.log``. + Kibana ~~~~~~ diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index 5b34dcc24b..015560765c 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -246,6 +246,7 @@ #enable_designate: "no" #enable_destroy_images: "no" #enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_osprofiler | bool or enable_skydive | bool or enable_monasca | bool else 'no' }}" +#enable_elasticsearch_curator: "no" #enable_etcd: "no" #enable_fluentd: "yes" #enable_freezer: "no" diff --git a/releasenotes/notes/add-elasticsearch-curator-88089d04f7ccd549.yaml b/releasenotes/notes/add-elasticsearch-curator-88089d04f7ccd549.yaml new file mode 100644 index 0000000000..79bd070ca8 --- /dev/null +++ b/releasenotes/notes/add-elasticsearch-curator-88089d04f7ccd549.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Adds Elasticsearch Curator for managing aggregated log data.