Install logstash 2.0
This makes a whole host of changes. First we install logstash from the deb packages that are now available. This package sets up the logstash user, group, logrotate for log files, a conf.d dir, and an init script for a service for us. This means all of that stuff can be removed from puppet and be provided by the package instead. In addition to cleanup related to the package we remove things that no longer make sense with newer logstash. There is no more logstash web service for example use kibana instead. Also the distinction between agent and indexer is no longer strong and logstash has a separate shipper tool which should be used in place of the agent (though backward compat for the agent class is preserved by setting it up as an indexer using an agent config). Depends-On: I2354fbe9d3ab25134c52bfe58f562dfdf9ff6786 Change-Id: If3fa423a80a03d09993e58f6d68a3a212b457468
This commit is contained in:
Clark Boylan
parent
4a8f77c2fb
commit
f326d9af28
|
|
@ -1,24 +0,0 @@
|
|||
# logstash - agent instance
|
||||
#
|
||||
# Copied from http://cookbook.logstash.net/recipes/using-upstart/
|
||||
|
||||
description "logstash agent instance"
|
||||
|
||||
start on virtual-filesystems
|
||||
stop on runlevel [06]
|
||||
|
||||
# Respawn it if the process exits
|
||||
respawn
|
||||
respawn limit 5 30
|
||||
expect fork
|
||||
|
||||
# You need to chdir somewhere writable because logstash needs to unpack a few
|
||||
# temporary files on startup.
|
||||
chdir /opt/logstash
|
||||
|
||||
script
|
||||
|
||||
# This runs logstash agent as the 'logstash' user
|
||||
su -s /bin/sh -c 'exec "$0" "$@"' logstash -- /usr/bin/java -jar logstash.jar agent -f /etc/logstash/agent.conf --log /var/log/logstash/agent.log &
|
||||
emit logstash-agent-running
|
||||
end script
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
# logstash - indexer instance
|
||||
#
|
||||
# Adapted from http://cookbook.logstash.net/recipes/using-upstart/
|
||||
|
||||
description "logstash indexer instance"
|
||||
|
||||
start on (virtual-filesystems and started networking)
|
||||
stop on runlevel [06]
|
||||
|
||||
# Respawn it if the process exits
|
||||
respawn
|
||||
respawn limit 5 30
|
||||
|
||||
# You need to chdir somewhere writable because logstash needs to unpack a few
|
||||
# temporary files on startup.
|
||||
chdir /opt/logstash
|
||||
|
||||
setuid logstash
|
||||
|
||||
script
|
||||
JAVA_ARGS=''
|
||||
if [ -f /etc/default/logstash-indexer ] ;
|
||||
then
|
||||
. /etc/default/logstash-indexer
|
||||
fi
|
||||
exec /usr/bin/java $JAVA_ARGS -jar logstash.jar agent -f /etc/logstash/indexer.conf --log /var/log/logstash/indexer.log
|
||||
end script
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
# logstash - web instance
|
||||
#
|
||||
# Copied from http://cookbook.logstash.net/recipes/using-upstart/
|
||||
|
||||
description "logstash web instance"
|
||||
|
||||
start on virtual-filesystems
|
||||
stop on runlevel [06]
|
||||
|
||||
# Respawn it if the process exits
|
||||
respawn
|
||||
respawn limit 5 30
|
||||
expect fork
|
||||
|
||||
# You need to chdir somewhere writable because logstash needs to unpack a few
|
||||
# temporary files on startup.
|
||||
chdir /opt/logstash
|
||||
|
||||
script
|
||||
|
||||
# This runs logstash web as the 'logstash' user
|
||||
su -s /bin/sh -c 'exec "$0" "$@"' logstash -- /usr/bin/java -jar logstash.jar web --backend elasticsearch://127.0.0.1/ --log /var/log/logstash/web.log &
|
||||
emit logstash-web-running
|
||||
end script
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
##############################
|
||||
# Default settings for logstash
|
||||
###############################
|
||||
|
||||
# Override Java location
|
||||
#JAVACMD=/usr/bin/java
|
||||
|
||||
# Set a home directory
|
||||
#LS_HOME=/var/lib/logstash
|
||||
|
||||
# Arguments to pass to logstash agent
|
||||
# Single worker so multiline events work
|
||||
LS_OPTS="-w 1"
|
||||
|
||||
# Arguments to pass to java
|
||||
LS_HEAP_SIZE="2g"
|
||||
#LS_JAVA_OPTS="-Djava.io.tmpdir=$HOME"
|
||||
|
||||
# pidfiles aren't used for upstart; this is for sysv users.
|
||||
#LS_PIDFILE=/var/run/logstash.pid
|
||||
|
||||
# user id to be invoked as; for upstart: edit /etc/init/logstash.conf
|
||||
#LS_USER=logstash
|
||||
|
||||
# logstash logging
|
||||
#LS_LOG_FILE=/var/log/logstash/logstash.log
|
||||
#LS_USE_GC_LOGGING="true"
|
||||
|
||||
# logstash configuration directory
|
||||
#LS_CONF_DIR=/etc/logstash/conf.d
|
||||
|
||||
# Open file limit; cannot be overridden in upstart
|
||||
#LS_OPEN_FILES=16384
|
||||
|
||||
# Nice level
|
||||
#LS_NICE=19
|
||||
|
||||
# If this is set to 1, then when `stop` is called, if the process has
|
||||
# not exited within a reasonable time, SIGKILL will be sent next.
|
||||
# The default behavior is to simply log a message "program stop failed; still running"
|
||||
KILL_ON_STOP_TIMEOUT=0
|
||||
|
|
@ -25,32 +25,8 @@
|
|||
class logstash::agent (
|
||||
$conf_template = 'logstash/agent.conf.erb'
|
||||
) {
|
||||
include ::logstash
|
||||
|
||||
file { '/etc/init/logstash-agent.conf':
|
||||
ensure => present,
|
||||
source => 'puppet:///modules/logstash/logstash-agent.conf',
|
||||
replace => true,
|
||||
owner => 'root',
|
||||
}
|
||||
|
||||
file { '/etc/logstash/agent.conf':
|
||||
ensure => present,
|
||||
content => template($conf_template),
|
||||
replace => true,
|
||||
owner => 'logstash',
|
||||
group => 'logstash',
|
||||
mode => '0644',
|
||||
require => Class['logstash'],
|
||||
}
|
||||
|
||||
service { 'logstash-agent':
|
||||
ensure => running,
|
||||
enable => true,
|
||||
subscribe => File['/etc/logstash/agent.conf'],
|
||||
require => [
|
||||
Class['logstash'],
|
||||
File['/etc/init/logstash-agent.conf'],
|
||||
]
|
||||
warning('This class is deprecated and logstash::indexer should be used instead')
|
||||
class { '::logstash::indexer':
|
||||
conf_template => $conf_template,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,20 +20,13 @@
|
|||
#
|
||||
# [*conf_template*]
|
||||
# String. Path to indexer config template.
|
||||
# Default: 'logstash/agent.conf.erb'
|
||||
# Default: 'logstash/indexer.conf.erb'
|
||||
class logstash::indexer (
|
||||
$conf_template = 'logstash/indexer.conf.erb'
|
||||
) {
|
||||
include ::logstash
|
||||
|
||||
file { '/etc/init/logstash-indexer.conf':
|
||||
ensure => present,
|
||||
source => 'puppet:///modules/logstash/logstash-indexer.conf',
|
||||
replace => true,
|
||||
owner => 'root',
|
||||
}
|
||||
|
||||
file { '/etc/logstash/indexer.conf':
|
||||
file { '/etc/logstash/conf.d/indexer.conf':
|
||||
ensure => present,
|
||||
content => template($conf_template),
|
||||
replace => true,
|
||||
|
|
@ -43,27 +36,23 @@ class logstash::indexer (
|
|||
require => Class['logstash'],
|
||||
}
|
||||
|
||||
service { 'logstash-indexer':
|
||||
ensure => running,
|
||||
enable => true,
|
||||
subscribe => File['/etc/logstash/indexer.conf'],
|
||||
require => [
|
||||
Class['logstash'],
|
||||
File['/etc/init/logstash-indexer.conf'],
|
||||
]
|
||||
file { '/etc/default/logstash':
|
||||
ensure => present,
|
||||
source => 'puppet:///modules/logstash/logstash.default',
|
||||
replace => true,
|
||||
owner => 'logstash',
|
||||
group => 'logstash',
|
||||
mode => '0644',
|
||||
require => Class['logstash'],
|
||||
}
|
||||
|
||||
include ::logrotate
|
||||
logrotate::file { 'indexer.log':
|
||||
log => '/var/log/logstash/indexer.log',
|
||||
options => [
|
||||
'compress',
|
||||
'copytruncate',
|
||||
'missingok',
|
||||
'rotate 7',
|
||||
'daily',
|
||||
'notifempty',
|
||||
service { 'logstash':
|
||||
ensure => running,
|
||||
enable => true,
|
||||
subscribe => [
|
||||
File['/etc/logstash/conf.d/indexer.conf'],
|
||||
File['/etc/default/logstash'],
|
||||
],
|
||||
require => Service['logstash-indexer'],
|
||||
require => Class['logstash'],
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,64 +17,11 @@
|
|||
# Class to install common logstash items.
|
||||
#
|
||||
class logstash {
|
||||
group { 'logstash':
|
||||
ensure => present,
|
||||
}
|
||||
|
||||
user { 'logstash':
|
||||
ensure => present,
|
||||
comment => 'Logstash User',
|
||||
home => '/opt/logstash',
|
||||
gid => 'logstash',
|
||||
shell => '/bin/bash',
|
||||
membership => 'minimum',
|
||||
require => Group['logstash'],
|
||||
}
|
||||
|
||||
file { '/opt/logstash':
|
||||
ensure => directory,
|
||||
owner => 'logstash',
|
||||
group => 'logstash',
|
||||
mode => '0644',
|
||||
require => User['logstash'],
|
||||
}
|
||||
|
||||
exec { 'get_logstash_jar':
|
||||
command => 'wget https://download.elasticsearch.org/logstash/logstash/logstash-1.3.3-flatjar.jar -O /opt/logstash/logstash-1.3.3-flatjar.jar',
|
||||
path => '/bin:/usr/bin',
|
||||
creates => '/opt/logstash/logstash-1.3.3-flatjar.jar',
|
||||
require => File['/opt/logstash'],
|
||||
}
|
||||
|
||||
file { '/opt/logstash/logstash-1.3.3-flatjar.jar':
|
||||
ensure => present,
|
||||
owner => 'logstash',
|
||||
group => 'logstash',
|
||||
mode => '0644',
|
||||
require => [
|
||||
User['logstash'],
|
||||
Exec['get_logstash_jar'],
|
||||
]
|
||||
}
|
||||
|
||||
file { '/opt/logstash/logstash.jar':
|
||||
ensure => link,
|
||||
target => '/opt/logstash/logstash-1.3.3-flatjar.jar',
|
||||
require => File['/opt/logstash/logstash-1.3.3-flatjar.jar'],
|
||||
}
|
||||
|
||||
file { '/var/log/logstash':
|
||||
ensure => directory,
|
||||
owner => 'logstash',
|
||||
group => 'logstash',
|
||||
mode => '0644',
|
||||
}
|
||||
|
||||
file { '/etc/logstash':
|
||||
ensure => directory,
|
||||
owner => 'logstash',
|
||||
group => 'logstash',
|
||||
mode => '0644',
|
||||
archive { '/tmp/logstash_2.0.0-1_all.deb':
|
||||
source => 'https://download.elastic.co/logstash/logstash/packages/debian/logstash_2.0.0-1_all.deb',
|
||||
extract => false,
|
||||
checksum => '094b18c77d7c959c1203012983337d5249922290',
|
||||
checksum_type => 'sha1',
|
||||
}
|
||||
|
||||
if ! defined(Package['openjdk-7-jre-headless']) {
|
||||
|
|
@ -82,4 +29,14 @@ class logstash {
|
|||
ensure => present,
|
||||
}
|
||||
}
|
||||
|
||||
package { 'logstash':
|
||||
ensure => latest,
|
||||
source => '/tmp/logstash_2.0.0-1_all.deb',
|
||||
provider => 'dpkg',
|
||||
require => [
|
||||
Package['openjdk-7-jre-headless'],
|
||||
Archive['/tmp/logstash_2.0.0-1_all.deb'],
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@
|
|||
# Class to install redis
|
||||
#
|
||||
class logstash::redis {
|
||||
warning('This class is deprecated and should not be used')
|
||||
# TODO(clarkb): Access to redis should be controlled at a network level
|
||||
# (with iptables) and with client authentication. Put this in place before
|
||||
# opening redis to external clients.
|
||||
|
|
|
|||
|
|
@ -57,27 +57,9 @@ class logstash::web (
|
|||
ensure => present,
|
||||
}
|
||||
|
||||
include ::logstash
|
||||
|
||||
case $frontend {
|
||||
'internal': {
|
||||
file { '/etc/init/logstash-web.conf':
|
||||
ensure => present,
|
||||
source => 'puppet:///modules/logstash/logstash-web.conf',
|
||||
replace => true,
|
||||
owner => 'root',
|
||||
}
|
||||
|
||||
service { 'logstash-web':
|
||||
ensure => running,
|
||||
enable => true,
|
||||
require => [
|
||||
Class['logstash'],
|
||||
File['/etc/init/logstash-web.conf'],
|
||||
],
|
||||
}
|
||||
|
||||
$vhost = 'logstash/logstash.vhost.erb'
|
||||
fail('This stopped working when we moved to Kibana3 and there is no analog in logstash 2.0. Use kibana.')
|
||||
}
|
||||
|
||||
'kibana': {
|
||||
|
|
|
|||
Loading…
Reference in New Issue