From 7b0be2ec8d7a303cedb2c68ac5f4ec8bd62dac77 Mon Sep 17 00:00:00 2001
From: Swann Croiset <scroiset@mirantis.com>
Date: Thu, 24 Dec 2015 12:58:21 +0100
Subject: [PATCH] Avoid indexing Uuid and EnvVersion fields

These fields compose every Heka message but are useless for end users.
The fix consists in explicitly enumarating fields to be indexed.

Change-Id: I55aa3719dad7d6b300b81aa0409d712f594c2e1e
---
 .../modules/heka/manifests/encoder/es_json.pp |  5 +++
 .../spec/defines/heka_encoder_es_json_spec.rb | 32 +++++++++++++++++++
 .../heka/templates/encoder/es_json.toml.erb   |  3 ++
 .../lma_collector/manifests/elasticsearch.pp  |  1 +
 .../modules/lma_collector/manifests/params.pp |  1 +
 5 files changed, 42 insertions(+)
 create mode 100644 deployment_scripts/puppet/modules/heka/spec/defines/heka_encoder_es_json_spec.rb

diff --git a/deployment_scripts/puppet/modules/heka/manifests/encoder/es_json.pp b/deployment_scripts/puppet/modules/heka/manifests/encoder/es_json.pp
index 5a1743b6f..7783db2b1 100644
--- a/deployment_scripts/puppet/modules/heka/manifests/encoder/es_json.pp
+++ b/deployment_scripts/puppet/modules/heka/manifests/encoder/es_json.pp
@@ -17,10 +17,15 @@ define heka::encoder::es_json (
   $es_index_from_timestamp = false,
   $index = undef,
   $ensure = present,
+  $fields = undef,
 ) {
 
   include heka::params
 
+  if $fields != undef {
+    validate_array($fields)
+  }
+
   file { "${config_dir}/encoder-${title}.toml":
     ensure  => $ensure,
     content => template('heka/encoder/es_json.toml.erb'),
diff --git a/deployment_scripts/puppet/modules/heka/spec/defines/heka_encoder_es_json_spec.rb b/deployment_scripts/puppet/modules/heka/spec/defines/heka_encoder_es_json_spec.rb
new file mode 100644
index 000000000..c3ccfbf98
--- /dev/null
+++ b/deployment_scripts/puppet/modules/heka/spec/defines/heka_encoder_es_json_spec.rb
@@ -0,0 +1,32 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+require 'spec_helper'
+
+describe 'heka::encoder::es_json' do
+    let(:title) { :es }
+    let(:facts) do
+        {:kernel => 'Linux', :operatingsystem => 'Ubuntu',
+         :osfamily => 'Debian'}
+    end
+
+    describe 'with title = es' do
+        let(:params) {{:config_dir => '/etc/hekad', :index => 'bar'}}
+        it { is_expected.to contain_file('/etc/hekad/encoder-es.toml') }
+    end
+    describe 'with fields list' do
+        let(:params) {{:config_dir => '/etc/hekad', :index => 'bar', :fields => ['foo', 'bar']}}
+        it { is_expected.to contain_file('/etc/hekad/encoder-es.toml').with_content(/fields = \[ "bar", "foo" \]/) }
+    end
+end
+
diff --git a/deployment_scripts/puppet/modules/heka/templates/encoder/es_json.toml.erb b/deployment_scripts/puppet/modules/heka/templates/encoder/es_json.toml.erb
index 0dbcc2668..75c1457a0 100644
--- a/deployment_scripts/puppet/modules/heka/templates/encoder/es_json.toml.erb
+++ b/deployment_scripts/puppet/modules/heka/templates/encoder/es_json.toml.erb
@@ -2,3 +2,6 @@
 type = "ESJsonEncoder"
 index = "<%= @index %>"
 es_index_from_timestamp = <%= @es_index_from_timestamp %>
+<% if @fields -%>
+fields = [ <%= @fields.sort.collect{ |x| '"%s"' % x }.join(", ") %> ]
+<% end -%>
diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/elasticsearch.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/elasticsearch.pp
index 830151cb2..9e38afb5b 100644
--- a/deployment_scripts/puppet/modules/lma_collector/manifests/elasticsearch.pp
+++ b/deployment_scripts/puppet/modules/lma_collector/manifests/elasticsearch.pp
@@ -24,6 +24,7 @@ class lma_collector::elasticsearch (
     config_dir              => $lma_collector::params::config_dir,
     index                   => '%{Type}-%{%Y.%m.%d}',
     es_index_from_timestamp => true,
+    fields                  => $lma_collector::params::elasticsearch_fields,
     notify                  => Class['lma_collector::service'],
   }
 
diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/params.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/params.pp
index 21ca82722..4591dbc64 100644
--- a/deployment_scripts/puppet/modules/lma_collector/manifests/params.pp
+++ b/deployment_scripts/puppet/modules/lma_collector/manifests/params.pp
@@ -142,6 +142,7 @@ class lma_collector::params {
 
   $elasticsearch_server = false
   $elasticsearch_port = '9200'
+  $elasticsearch_fields = ['Timestamp', 'Type', 'Logger', 'Severity', 'Payload', 'Pid', 'Hostname', 'DynamicFields']
 
   $influxdb_server = false
   $influxdb_port = '8086'