Add support for data curation

This change configures a cron job that launches the curator script to
clean up old indices from Elasticsearch. It also configures the number
of replicas to keep per index. Note the value is always zero for now
since the plugin doesn't support cluster deployment yet.

Change-Id: Id22ebb949aeda9c90a05a975152762e2fe4b7eea

deployment_scripts/puppet/manifests/elasticsearch.pp

@@ -53,10 +53,12 @@ if $elasticsearch_kibana['node_name'] == hiera('user_node_name') {
   }
 
   lma_logging_analytics::es_template { ['log', 'notification']:
-    require => Elasticsearch::Instance[$es_instance],
+    number_of_replicas => 0 + $elasticsearch_kibana['number_of_replicas'],
+    require            => Elasticsearch::Instance[$es_instance],
   }
 
-  package { 'python-elasticsearch-curator':
-    ensure => installed,
+  class { 'lma_logging_analytics::curator':
+    retention_period => $elasticsearch_kibana['retention_period'],
+    prefixes         => ['log', 'notification'],
   }
 }

deployment_scripts/puppet/manifests/kibana.pp

@@ -15,5 +15,7 @@
 $elasticsearch_kibana = hiera('elasticsearch_kibana')
 
 if $elasticsearch_kibana['node_name'] == hiera('user_node_name') {
-  class { 'lma_logging_analytics::kibana': }
+  class { 'lma_logging_analytics::kibana':
+    number_of_replicas => 0 + $elasticsearch_kibana['number_of_replicas']
+  }
 }

deployment_scripts/puppet/modules/lma_logging_analytics/manifests/curator.pp

@@ -0,0 +1,42 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+class lma_logging_analytics::curator (
+  $retention_period = $lma_logging_analytics::params::retention_period,
+  $prefixes = $lma_logging_analytics::params::indexes_prefixes,
+) inherits lma_logging_analytics::params {
+
+  validate_integer($retention_period)
+  validate_array($prefixes)
+
+  package { 'python-elasticsearch-curator':
+    ensure => installed,
+  }
+
+  if size($prefixes) > 0 and $retention_period > 0 {
+    # Timestamps are UTC-based but the end-user may be in a different timezone.
+    # Lets add 1 day to the given retention period to make sure that we don't
+    # drop indices too early.
+    $real_retention_period = 1 + $retention_period
+    $regex = join($prefixes, '|')
+    cron { 'es-curator':
+      ensure   => present,
+      command  => "/usr/local/bin/curator --host localhost --port 9200 --debug delete indices --regex '^(${regex})-.*$' --time-unit days --older-than ${real_retention_period} --timestring \"%Y.%m.%d\"",
+      minute   => '0',
+      hour     => '2',
+      month    => '*',
+      monthday => '*',
+    }
+  }
+}

deployment_scripts/puppet/modules/lma_logging_analytics/manifests/kibana.pp

@@ -15,8 +15,11 @@
 # Class lma_logging_analytics::kibana
 
 
-class lma_logging_analytics::kibana {
-  include lma_logging_analytics::params
+class lma_logging_analytics::kibana (
+  $number_of_replicas = $lma_logging_analytics::params::kibana_replicas,
+) inherits lma_logging_analytics::params {
+
+  validate_integer($number_of_replicas)
 
   $kibana_dir    = $lma_logging_analytics::params::kibana_dir
   $kibana_conf   = $lma_logging_analytics::params::kibana_config
@@ -42,7 +45,7 @@ class lma_logging_analytics::kibana {
   }
 
   elasticsearch::template { 'kibana':
-    content => '{"template":"kibana-*", "settings": {"number_of_replicas":0}}'
+    content => "{\"template\":\"kibana-*\", \"settings\": {\"number_of_replicas\":${number_of_replicas}}}"
   }
 
   # Note that the dashboards are stored in templates/ because it is the only way

deployment_scripts/puppet/modules/lma_logging_analytics/manifests/params.pp

@@ -15,9 +15,13 @@
 # Class lma_logging_analytics::params
 
 class lma_logging_analytics::params {
+  $retention_period        = 30
+  $indexes_prefixes        = []
+
   $kibana_dir              = '/opt/kibana'
   $kibana_config           = "${kibana_dir}/config.js"
   $kibana_dashboard_dir    = "${kibana_dir}/app/dashboards"
   $kibana_dashboard_prefix = 'Logging, Monitoring and Alerting - '
   $kibana_default_route    = join(['/dashboard/elasticsearch/', $kibana_dashboard_prefix, 'Logs'], '')
+  $kibana_replicas         = 0
 }

environment_config.yaml

@@ -1,5 +1,27 @@
 attributes:
 
+  retention_period:
+    value: '30'
+    label: 'Retention period'
+    description: 'The number of days after which data is automatically deleted within the Elasticsearch system (0 to never delete data).'
+    weight: 5
+    type: "text"
+    regex: &number_validation
+        source: '^\d+$'
+        error: "You must provide a number"
+
+  number_of_replicas:
+    value: '0'
+    label: 'Replication factor'
+    description: 'The number of replicas'
+    weight: 6
+    type: "text"
+    regex: *number_validation
+    # Don't expose the replication factor until clustering is supported
+    restrictions:
+      - condition: "true"
+        action: hide
+
   node_name:
     value: 'elasticsearch'
     label: 'Node name'