Properly support logstash conf.d

Add in proper support for conf.d configs. Create input at 00-input,
output at 99-output, and filters at user specifiable levels in between
using a new defined type for filters.

Co-Authored-By: Jonathan Harker <code@gentlydownthe.net>
Change-Id: Icbca7a6ba0c5a94a273ef158f707311b588483fd

manifests/filter.pp

@@ -0,0 +1,34 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# = Class: logstash::filter
+#
+# Class to install logstash filter configs
+#
+# == Parameters
+#
+# [*level*]
+#   String. Conf level prefix for stitching files together in order.
+# [*target*]
+#   String. Path to actual config location will be symlinked to.
+define logstash::filter (
+  $level,
+  $target,
+) {
+  include ::logstash
+
+  file { "/etc/logstash/conf.d/${level}-${name}.conf":
+    ensure  => link,
+    target  => $target,
+    require => Class['logstash'],
+  }
+}

manifests/indexer.pp

@@ -19,21 +19,56 @@
 # == Parameters
 #
 # [*conf_template*]
-#   String. Path to indexer config template.
-#   Default: 'logstash/indexer.conf.erb'
+#   String. (deprecated) Path to indexer config template.
+#   Default: undef
+# [*input_template*]
+#   String. Path to indexer input config template.
+#   Default: 'logstash/input.conf.erb'
+# [*output_template*]
+#   String. Path to indexer output config template.
+#   Default: 'logstash/output.conf.erb'
 class logstash::indexer (
-  $conf_template = 'logstash/indexer.conf.erb',
+  $conf_template   = undef,
+  $input_template  = 'logstash/input.conf.erb',
+  $output_template = 'logstash/output.conf.erb',
 ) {
   include ::logstash
 
-  file { '/etc/logstash/conf.d/indexer.conf':
-    ensure  => present,
-    content => template($conf_template),
-    replace => true,
-    owner   => 'logstash',
-    group   => 'logstash',
-    mode    => '0644',
-    require => Class['logstash'],
+  if $conf_template != undef {
+    notify { 'Using $conf_template is deprecated, please switch to $input_template, $output_template and ::logstash::filter defines.': }
+
+    file { '/etc/logstash/conf.d/indexer.conf':
+      ensure  => present,
+      content => template($conf_template),
+      replace => true,
+      owner   => 'logstash',
+      group   => 'logstash',
+      mode    => '0644',
+      require => Class['logstash'],
+      notify  => Service['logstash'],
+    }
+  } else {
+    file { '/etc/logstash/conf.d/00-input.conf':
+      ensure  => present,
+      content => template($input_template),
+      replace => true,
+      owner   => 'logstash',
+      group   => 'logstash',
+      mode    => '0644',
+      require => Class['logstash'],
+      notify  => Service['logstash'],
+    }
+
+    file { '/etc/logstash/conf.d/99-output.conf':
+      ensure  => present,
+      content => template($output_template),
+      replace => true,
+      owner   => 'logstash',
+      group   => 'logstash',
+      mode    => '0644',
+      require => Class['logstash'],
+      notify  => Service['logstash'],
+    }
   }
 
   file { '/etc/default/logstash':
@@ -50,7 +85,6 @@ class logstash::indexer (
     ensure    => running,
     enable    => true,
     subscribe => [
-      File['/etc/logstash/conf.d/indexer.conf'],
       File['/etc/default/logstash'],
     ],
     require   => Class['logstash'],

spec/acceptance/basic_spec.rb

@@ -30,7 +30,7 @@ describe 'puppet-logstash module', :if => ['debian', 'ubuntu'].include?(os[:fami
     end
 
     describe 'module logstash::indexer' do
-      describe file('/etc/logstash/conf.d/indexer.conf') do
+      describe file('/etc/logstash/conf.d/00-input.conf') do
         it { should be_file }
         it { should be_owned_by 'logstash' }
         it { should be_grouped_into 'logstash' }
@@ -38,6 +38,13 @@ describe 'puppet-logstash module', :if => ['debian', 'ubuntu'].include?(os[:fami
         its(:content) { should include 'host => "127.0.0.1"' }
       end
 
+      describe file('/etc/logstash/conf.d/99-output.conf') do
+        it { should be_file }
+        it { should be_owned_by 'logstash' }
+        it { should be_grouped_into 'logstash' }
+        its(:content) { should include 'host => "127.0.0.1"' }
+      end
+
       describe file('/etc/default/logstash') do
         its(:content) { should include 'LS_OPTS="-w 1"' }
       end

templates/input.conf.erb

@@ -0,0 +1,12 @@
+input {
+  redis {
+    host => "127.0.0.1"
+    type => "redis-input"
+    # these settings should match the output of the agent
+    data_type => "list"
+    key => "logstash"
+
+    # We use json_event here since the sender is a logstash agent
+    format => "json_event"
+  }
+}

templates/output.conf.erb

@@ -0,0 +1,5 @@
+output {
+  elasticsearch {
+    host => "127.0.0.1"
+  }
+}