Merge "Install logstash 2.0"

files/logstash-agent.conf

@@ -1,24 +0,0 @@
-# logstash - agent instance
-#
-# Copied from http://cookbook.logstash.net/recipes/using-upstart/
-
-description     "logstash agent instance"
-
-start on virtual-filesystems
-stop on runlevel [06]
-
-# Respawn it if the process exits
-respawn
-respawn limit 5 30
-expect fork
-
-# You need to chdir somewhere writable because logstash needs to unpack a few
-# temporary files on startup.
-chdir /opt/logstash
-
-script
-
-  # This runs logstash agent as the 'logstash' user
-  su -s /bin/sh -c 'exec "$0" "$@"' logstash -- /usr/bin/java -jar logstash.jar agent -f /etc/logstash/agent.conf --log /var/log/logstash/agent.log &
-  emit logstash-agent-running
-end script

files/logstash-indexer.conf

@@ -1,27 +0,0 @@
-# logstash - indexer instance
-#
-# Adapted from http://cookbook.logstash.net/recipes/using-upstart/
-
-description     "logstash indexer instance"
-
-start on (virtual-filesystems and started networking)
-stop on runlevel [06]
-
-# Respawn it if the process exits
-respawn
-respawn limit 5 30
-
-# You need to chdir somewhere writable because logstash needs to unpack a few
-# temporary files on startup.
-chdir /opt/logstash
-
-setuid logstash
-
-script
-  JAVA_ARGS=''
-  if [ -f /etc/default/logstash-indexer ] ;
-  then
-    . /etc/default/logstash-indexer
-  fi
-  exec /usr/bin/java $JAVA_ARGS -jar logstash.jar agent -f /etc/logstash/indexer.conf --log /var/log/logstash/indexer.log
-end script

files/logstash-web.conf

@@ -1,24 +0,0 @@
-# logstash - web instance
-#
-# Copied from http://cookbook.logstash.net/recipes/using-upstart/
-
-description     "logstash web instance"
-
-start on virtual-filesystems
-stop on runlevel [06]
-
-# Respawn it if the process exits
-respawn
-respawn limit 5 30
-expect fork
-
-# You need to chdir somewhere writable because logstash needs to unpack a few
-# temporary files on startup.
-chdir /opt/logstash
-
-script
-
-  # This runs logstash web as the 'logstash' user
-  su -s /bin/sh -c 'exec "$0" "$@"' logstash -- /usr/bin/java -jar logstash.jar web --backend elasticsearch://127.0.0.1/ --log /var/log/logstash/web.log &
-  emit logstash-web-running
-end script

files/logstash.default

@@ -0,0 +1,41 @@
+##############################
+# Default settings for logstash
+###############################
+
+# Override Java location
+#JAVACMD=/usr/bin/java
+
+# Set a home directory
+#LS_HOME=/var/lib/logstash
+
+# Arguments to pass to logstash agent
+# Single worker so multiline events work
+LS_OPTS="-w 1"
+
+# Arguments to pass to java
+LS_HEAP_SIZE="2g"
+#LS_JAVA_OPTS="-Djava.io.tmpdir=$HOME"
+
+# pidfiles aren't used for upstart; this is for sysv users.
+#LS_PIDFILE=/var/run/logstash.pid
+
+# user id to be invoked as; for upstart: edit /etc/init/logstash.conf
+#LS_USER=logstash
+
+# logstash logging
+#LS_LOG_FILE=/var/log/logstash/logstash.log
+#LS_USE_GC_LOGGING="true"
+
+# logstash configuration directory
+#LS_CONF_DIR=/etc/logstash/conf.d
+
+# Open file limit; cannot be overridden in upstart
+#LS_OPEN_FILES=16384
+
+# Nice level
+#LS_NICE=19
+
+# If this is set to 1, then when `stop` is called, if the process has
+# not exited within a reasonable time, SIGKILL will be sent next.
+# The default behavior is to simply log a message "program stop failed; still running"
+KILL_ON_STOP_TIMEOUT=0

manifests/agent.pp

@@ -25,32 +25,8 @@
 class logstash::agent (
   $conf_template = 'logstash/agent.conf.erb'
 ) {
-  include ::logstash
+  warning('This class is deprecated and logstash::indexer should be used instead')
-
+  class { '::logstash::indexer':
-  file { '/etc/init/logstash-agent.conf':
+    conf_template => $conf_template,
-    ensure  => present,
-    source  => 'puppet:///modules/logstash/logstash-agent.conf',
-    replace => true,
-    owner   => 'root',
-  }
-
-  file { '/etc/logstash/agent.conf':
-    ensure  => present,
-    content => template($conf_template),
-    replace => true,
-    owner   => 'logstash',
-    group   => 'logstash',
-    mode    => '0644',
-    require => Class['logstash'],
-  }
-
-  service { 'logstash-agent':
-    ensure    => running,
-    enable    => true,
-    subscribe => File['/etc/logstash/agent.conf'],
-    require   => [
-      Class['logstash'],
-      File['/etc/init/logstash-agent.conf'],
-    ]
   }
 }

manifests/indexer.pp

@@ -20,20 +20,13 @@
 #
 # [*conf_template*]
 #   String. Path to indexer config template.
-#   Default: 'logstash/agent.conf.erb'
+#   Default: 'logstash/indexer.conf.erb'
 class logstash::indexer (
   $conf_template = 'logstash/indexer.conf.erb'
 ) {
   include ::logstash
 
-  file { '/etc/init/logstash-indexer.conf':
+  file { '/etc/logstash/conf.d/indexer.conf':
-    ensure  => present,
-    source  => 'puppet:///modules/logstash/logstash-indexer.conf',
-    replace => true,
-    owner   => 'root',
-  }
-
-  file { '/etc/logstash/indexer.conf':
     ensure  => present,
     content => template($conf_template),
     replace => true,
@@ -43,27 +36,23 @@ class logstash::indexer (
     require => Class['logstash'],
   }
 
-  service { 'logstash-indexer':
+  file { '/etc/default/logstash':
-    ensure    => running,
+    ensure  => present,
-    enable    => true,
+    source  => 'puppet:///modules/logstash/logstash.default',
-    subscribe => File['/etc/logstash/indexer.conf'],
+    replace => true,
-    require   => [
+    owner   => 'logstash',
-      Class['logstash'],
+    group   => 'logstash',
-      File['/etc/init/logstash-indexer.conf'],
+    mode    => '0644',
-    ]
+    require => Class['logstash'],
   }
 
-  include ::logrotate
+  service { 'logstash':
-  logrotate::file { 'indexer.log':
+    ensure    => running,
-    log     => '/var/log/logstash/indexer.log',
+    enable    => true,
-    options => [
+    subscribe => [
-      'compress',
+      File['/etc/logstash/conf.d/indexer.conf'],
-      'copytruncate',
+      File['/etc/default/logstash'],
-      'missingok',
-      'rotate 7',
-      'daily',
-      'notifempty',
     ],
-    require => Service['logstash-indexer'],
+    require   => Class['logstash'],
   }
 }

manifests/init.pp

@@ -17,64 +17,11 @@
 # Class to install common logstash items.
 #
 class logstash {
-  group { 'logstash':
+  archive { '/tmp/logstash_2.0.0-1_all.deb':
-    ensure => present,
+    source        => 'https://download.elastic.co/logstash/logstash/packages/debian/logstash_2.0.0-1_all.deb',
-  }
+    extract       => false,
-
+    checksum      => '094b18c77d7c959c1203012983337d5249922290',
-  user { 'logstash':
+    checksum_type => 'sha1',
-    ensure     => present,
-    comment    => 'Logstash User',
-    home       => '/opt/logstash',
-    gid        => 'logstash',
-    shell      => '/bin/bash',
-    membership => 'minimum',
-    require    => Group['logstash'],
-  }
-
-  file { '/opt/logstash':
-    ensure  => directory,
-    owner   => 'logstash',
-    group   => 'logstash',
-    mode    => '0644',
-    require => User['logstash'],
-  }
-
-  exec { 'get_logstash_jar':
-    command => 'wget https://download.elasticsearch.org/logstash/logstash/logstash-1.3.3-flatjar.jar -O /opt/logstash/logstash-1.3.3-flatjar.jar',
-    path    => '/bin:/usr/bin',
-    creates => '/opt/logstash/logstash-1.3.3-flatjar.jar',
-    require => File['/opt/logstash'],
-  }
-
-  file { '/opt/logstash/logstash-1.3.3-flatjar.jar':
-    ensure  => present,
-    owner   => 'logstash',
-    group   => 'logstash',
-    mode    => '0644',
-    require => [
-      User['logstash'],
-      Exec['get_logstash_jar'],
-    ]
-  }
-
-  file { '/opt/logstash/logstash.jar':
-    ensure  => link,
-    target  => '/opt/logstash/logstash-1.3.3-flatjar.jar',
-    require => File['/opt/logstash/logstash-1.3.3-flatjar.jar'],
-  }
-
-  file { '/var/log/logstash':
-    ensure => directory,
-    owner  => 'logstash',
-    group  => 'logstash',
-    mode   => '0644',
-  }
-
-  file { '/etc/logstash':
-    ensure => directory,
-    owner  => 'logstash',
-    group  => 'logstash',
-    mode   => '0644',
   }
 
   if ! defined(Package['openjdk-7-jre-headless']) {
@@ -82,4 +29,14 @@ class logstash {
       ensure => present,
     }
   }
+
+  package { 'logstash':
+    ensure   => latest,
+    source   => '/tmp/logstash_2.0.0-1_all.deb',
+    provider => 'dpkg',
+    require  => [
+      Package['openjdk-7-jre-headless'],
+      Archive['/tmp/logstash_2.0.0-1_all.deb'],
+    ]
+  }
 }

manifests/redis.pp

@@ -18,6 +18,7 @@
 # Class to install redis
 #
 class logstash::redis {
+  warning('This class is deprecated and should not be used')
   # TODO(clarkb): Access to redis should be controlled at a network level
   # (with iptables) and with client authentication. Put this in place before
   # opening redis to external clients.

manifests/web.pp

@@ -57,27 +57,9 @@ class logstash::web (
     ensure => present,
   }
 
-  include ::logstash
-
   case $frontend {
     'internal': {
-      file { '/etc/init/logstash-web.conf':
+      fail('This stopped working when we moved to Kibana3 and there is no analog in logstash 2.0. Use kibana.')
-        ensure  => present,
-        source  => 'puppet:///modules/logstash/logstash-web.conf',
-        replace => true,
-        owner   => 'root',
-      }
-
-      service { 'logstash-web':
-        ensure  => running,
-        enable  => true,
-        require => [
-          Class['logstash'],
-          File['/etc/init/logstash-web.conf'],
-        ],
-      }
-
-      $vhost = 'logstash/logstash.vhost.erb'
     }
 
     'kibana': {