Support deploying Elasticsearch Curator

This allows users to supply an Elasticsearch Curator actions file
to manage log retention [1]. Curator then runs on a cron job, which
defaults to every day. A default curator actions file is provided,
which can be customised by the end user if required.

[1] https://www.elastic.co/guide/en/elasticsearch/client/curator/current/actionfile.html

Change-Id: Ide9baea9190ae849e61b9d8b6cff3305bdcdd534

ansible/group_vars/all.yml

@@ -724,6 +724,11 @@ skip_stop_containers: []
 
 elasticsearch_address: "{{ kolla_internal_fqdn }}"
 enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_osprofiler | bool or enable_skydive | bool or enable_monasca | bool else 'no' }}"
+
+# If using Curator an actions file will need to be defined. Please see
+# the documentation.
+enable_elasticsearch_curator: "no"
+
 enable_kibana: "{{ 'yes' if enable_central_logging | bool or enable_monasca | bool else 'no' }}"
 
 ####################

ansible/inventory/all-in-one

@@ -259,6 +259,10 @@ control
 # function appropriately. For example, neutron-metadata-agent must run on the
 # same host as the l3-agent and (depending on configuration) the dhcp-agent.
 
+# Elasticsearch Curator
+[elasticsearch-curator:children]
+elasticsearch
+
 # Glance
 [glance-api:children]
 glance

ansible/inventory/multinode

@@ -278,6 +278,10 @@ control
 # function appropriately. For example, neutron-metadata-agent must run on the
 # same host as the l3-agent and (depending on configuration) the dhcp-agent.
 
+# Elasticsearch Curator
+[elasticsearch-curator:children]
+elasticsearch
+
 # Glance
 [glance-api:children]
 glance

ansible/roles/elasticsearch/defaults/main.yml

@@ -19,6 +19,13 @@ elasticsearch_services:
         port: "{{ elasticsearch_port }}"
         frontend_http_extra:
           - "option dontlog-normal"
+  elasticsearch-curator:
+    container_name: elasticsearch_curator
+    group: elasticsearch-curator
+    enabled: "{{ enable_elasticsearch_curator }}"
+    image: "{{ elasticsearch_curator_image_full }}"
+    volumes: "{{ elasticsearch_curator_default_volumes + elasticsearch_curator_extra_volumes }}"
+    dimensions: "{{ elasticsearch_curator_dimensions }}"
 
 
 ####################
@@ -28,6 +35,33 @@ elasticsearch_cluster_name: "kolla_logging"
 es_heap_size: "1g"
 es_java_opts: "{% if es_heap_size %}-Xms{{ es_heap_size }} -Xmx{{ es_heap_size }}{%endif%}"
 
+#######################
+# Elasticsearch Curator
+#######################
+
+# How frequently Curator runs. On multinode deployments of Curator
+# you may wish to override this in hostvars so that Curator does
+# not run simultaneously on all nodes. Defaults to every midnight.
+elasticsearch_curator_cron_schedule: "0 0 * * *"
+
+# When set to True, Curator will not modify Elasticsearch data, but
+# will print what it *would* do to the Curator log file. This is a
+# useful way of checking that Curator actions are working as expected.
+elasticsearch_curator_dry_run: false
+
+# Index prefix pattern. Any indices matching this regex will
+# be managed by Curator.
+elasticsearch_curator_index_pattern: "^{{ 'monasca' if enable_monasca|bool else kibana_log_prefix }}-.*"
+
+# Duration after which an index is staged for deletion. This is
+# implemented by closing the index. Whilst in this state the index
+# contributes negligible load on the cluster and may be manually
+# re-opened if required.
+elasticsearch_curator_soft_retention_period_days: 30
+
+# Duration after which an index is permanently erased from the cluster.
+elasticsearch_curator_hard_retention_period_days: 60
+
 ####################
 # Docker
 ####################
@@ -36,10 +70,21 @@ elasticsearch_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ d
 elasticsearch_tag: "{{ openstack_release }}"
 elasticsearch_image_full: "{{ elasticsearch_image }}:{{ elasticsearch_tag }}"
 
+elasticsearch_curator_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ elasticsearch_install_type }}-elasticsearch-curator"
+elasticsearch_curator_tag: "{{ openstack_release }}"
+elasticsearch_curator_image_full: "{{ elasticsearch_curator_image }}:{{ elasticsearch_curator_tag }}"
+
 elasticsearch_dimensions: "{{ default_container_dimensions }}"
+elasticsearch_curator_dimensions: "{{ default_container_dimensions }}"
 
 elasticsearch_default_volumes:
   - "{{ node_config_directory }}/elasticsearch/:{{ container_config_directory }}/"
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ elasticsearch_datadir_volume }}:/var/lib/elasticsearch/data"
+elasticsearch_curator_default_volumes:
+  - "{{ node_config_directory }}/elasticsearch-curator/:{{ container_config_directory }}/"
+  - "/etc/localtime:/etc/localtime:ro"
+  - "kolla_logs:/var/log/kolla"
+
 elasticsearch_extra_volumes: "{{ default_extra_volumes }}"
+elasticsearch_curator_extra_volumes: "{{ default_extra_volumes }}"

ansible/roles/elasticsearch/handlers/main.yml

@@ -14,3 +14,18 @@
     dimensions: "{{ service.dimensions }}"
   when:
     - kolla_action != "config"
+
+- name: Restart elasticsearch-curator container
+  vars:
+    service_name: "elasticsearch-curator"
+    service: "{{ elasticsearch_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"

ansible/roles/elasticsearch/tasks/check-containers.yml

@@ -8,7 +8,7 @@
     image: "{{ item.value.image }}"
     volumes: "{{ item.value.volumes }}"
     dimensions: "{{ item.value.dimensions }}"
-    environment: "{{ item.value.environment }}"
+    environment: "{{ item.value.environment|default(omit) }}"
   when:
     - inventory_hostname in groups[item.value.group]
     - item.value.enabled | bool

ansible/roles/elasticsearch/tasks/config.yml

@@ -34,9 +34,9 @@
   notify:
     - Restart {{ item.key }} container
 
-- name: Copying over elasticsearch.yml
+- name: Copying over elasticsearch service config files
   template:
-    src: "elasticsearch.yml.j2"
+    src: "{{ item.key }}.yml.j2"
     dest: "{{ node_config_directory }}/{{ item.key }}/{{ item.key }}.yml"
     mode: "0660"
   become: true
@@ -47,5 +47,36 @@
   notify:
     - Restart {{ item.key }} container
 
+- name: Copying over elasticsearch curator actions
+  vars:
+    service: "{{ elasticsearch_services['elasticsearch-curator'] }}"
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/elasticsearch-curator/elasticsearch-curator-actions.yml"
+    mode: "0660"
+  become: true
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  with_first_found:
+    - "{{ node_custom_config }}/elasticsearch/elasticsearch-curator-actions.yml"
+    - "{{ role_path }}/templates/elasticsearch-curator-actions.yml.j2"
+  notify:
+    - Restart elasticsearch-curator container
+
+- name: Copying over elasticsearch curator crontab
+  vars:
+    service: "{{ elasticsearch_services['elasticsearch-curator'] }}"
+  template:
+    src: "{{ role_path }}/templates/elasticsearch-curator.crontab.j2"
+    dest: "{{ node_config_directory }}/elasticsearch-curator/elasticsearch-curator.crontab"
+    mode: "0660"
+  become: true
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart elasticsearch-curator container
+
 - include_tasks: check-containers.yml
   when: kolla_action != "config"

ansible/roles/elasticsearch/tasks/pull.yml

@@ -1,5 +1,5 @@
 ---
-- name: Pulling elasticsearch image
+- name: Pulling elasticsearch images
   become: true
   kolla_docker:
     action: "pull_image"

ansible/roles/elasticsearch/templates/elasticsearch-curator-actions.yml.j2

@@ -0,0 +1,33 @@
+actions:
+  1:
+    action: close
+    description: >-
+      Closes indices
+    options:
+      ignore_empty_list: True
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: "{{ elasticsearch_curator_index_pattern }}"
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: "{{ elasticsearch_curator_soft_retention_period_days }}"
+  2:
+    action: delete_indices
+    description: >-
+      Delete indicies
+    options:
+      ignore_empty_list: True
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: "{{ elasticsearch_curator_index_pattern }}"
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: "{{ elasticsearch_curator_hard_retention_period_days }}"

ansible/roles/elasticsearch/templates/elasticsearch-curator.crontab.j2

@@ -0,0 +1 @@
+{{ elasticsearch_curator_cron_schedule }} curator --config /etc/elasticsearch-curator/curator.yml {% if elasticsearch_curator_dry_run|bool %}--dry-run {% endif %}/etc/elasticsearch-curator/actions.yml

ansible/roles/elasticsearch/templates/elasticsearch-curator.json.j2

@@ -0,0 +1,31 @@
+{% set cron_cmd = 'cron -f' if kolla_base_distro in ['ubuntu', 'debian'] else 'crond -s -n' %}
+{
+    "command": "{{ cron_cmd }}",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/elasticsearch-curator.crontab",
+            "dest": "/var/spool/cron/elasticsearch",
+            "owner": "root",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/elasticsearch-curator.yml",
+            "dest": "/etc/elasticsearch-curator/curator.yml",
+            "owner": "elasticsearch",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/elasticsearch-curator-actions.yml",
+            "dest": "/etc/elasticsearch-curator/actions.yml",
+            "owner": "elasticsearch",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/elasticsearch",
+            "owner": "elasticsearch:elasticsearch",
+            "recurse": true
+        }
+    ]
+}

ansible/roles/elasticsearch/templates/elasticsearch-curator.yml.j2

@@ -0,0 +1,8 @@
+client:
+  hosts: [{% for host in groups['elasticsearch'] %}"{{ 'api' | kolla_address(host) }}"{% if not loop.last %},{% endif %}{% endfor %}]
+  port: {{ elasticsearch_port }}
+  timeout: 30
+
+logging:
+  loglevel: INFO
+  logfile: /var/log/kolla/elasticsearch/elasticsearch-curator.log

doc/source/reference/logging-and-monitoring/central-logging-guide.rst

@@ -35,6 +35,40 @@ By default Elasticsearch is deployed on port ``9200``.
    ``elasticsearch`` to store the data of Elasticsearch. The path can be set via
    the variable ``elasticsearch_datadir_volume``.
 
+Curator
+-------
+
+To stop your disks filling up, retention policies can be set. These are
+enforced by Elasticsearch Curator which can be enabled by setting the
+following in ``/etc/kolla/globals.yml``:
+
+.. code-block:: yaml
+
+   enable_elasticsearch_curator: "yes"
+
+Elasticsearch Curator is configured via an actions file. The format of the
+actions file is described in the `Elasticsearch Curator documentation <https://www.elastic.co/guide/en/elasticsearch/client/curator/current/actionfile.html>`_.
+A default actions file is provided which closes indices and then deletes them
+some time later. The periods for these operations, as well as the prefix for
+determining which indicies should be managed are defined in the Elasticsearch
+role defaults and can be overridden in ``/etc/kolla/globals.yml`` if required.
+
+If the default actions file is not malleable enough, a custom actions file can
+be placed in the Kolla custom config directory, for example:
+``/etc/kolla/config/elasticsearch/elasticsearch-curator-actions.yml``.
+
+When testing the actions file you may wish to perform a dry run to be certain
+of what Curator will actually do. A dry run can be enabled by setting the
+following in ``/etc/kolla/globals.yml``:
+
+.. code-block:: yaml
+
+   elasticsearch_curator_dry_run: "yes"
+
+The actions which *would* be taken if a dry run were to be disabled are then
+logged in the Elasticsearch Kolla logs folder under
+``/var/log/kolla/elasticsearch/elasticsearch-curator.log``.
+
 Kibana
 ~~~~~~
 

etc/kolla/globals.yml

@@ -246,6 +246,7 @@
 #enable_designate: "no"
 #enable_destroy_images: "no"
 #enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_osprofiler | bool or enable_skydive | bool or enable_monasca | bool else 'no' }}"
+#enable_elasticsearch_curator: "no"
 #enable_etcd: "no"
 #enable_fluentd: "yes"
 #enable_freezer: "no"

releasenotes/notes/add-elasticsearch-curator-88089d04f7ccd549.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Adds Elasticsearch Curator for managing aggregated log data.