From f326d9af2844ddfa9f2d33d912f109bd140d7a52 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Thu, 19 Nov 2015 11:35:38 -0800 Subject: [PATCH] Install logstash 2.0 This makes a whole host of changes. First we install logstash from the deb packages that are now available. This package sets up the logstash user, group, logrotate for log files, a conf.d dir, and an init script for a service for us. This means all of that stuff can be removed from puppet and be provided by the package instead. In addition to cleanup related to the package we remove things that no longer make sense with newer logstash. There is no more logstash web service for example use kibana instead. Also the distinction between agent and indexer is no longer strong and logstash has a separate shipper tool which should be used in place of the agent (though backward compat for the agent class is preserved by setting it up as an indexer using an agent config). Depends-On: I2354fbe9d3ab25134c52bfe58f562dfdf9ff6786 Change-Id: If3fa423a80a03d09993e58f6d68a3a212b457468 --- files/logstash-agent.conf | 24 ------------ files/logstash-indexer.conf | 27 -------------- files/logstash-web.conf | 24 ------------ files/logstash.default | 41 +++++++++++++++++++++ manifests/agent.pp | 30 ++------------- manifests/indexer.pp | 45 +++++++++-------------- manifests/init.pp | 73 ++++++++----------------------------- manifests/redis.pp | 1 + manifests/web.pp | 20 +--------- 9 files changed, 78 insertions(+), 207 deletions(-) delete mode 100644 files/logstash-agent.conf delete mode 100644 files/logstash-indexer.conf delete mode 100644 files/logstash-web.conf create mode 100644 files/logstash.default diff --git a/files/logstash-agent.conf b/files/logstash-agent.conf deleted file mode 100644 index ba8c978..0000000 --- a/files/logstash-agent.conf +++ /dev/null @@ -1,24 +0,0 @@ -# logstash - agent instance -# -# Copied from http://cookbook.logstash.net/recipes/using-upstart/ - -description "logstash agent instance" - -start on virtual-filesystems -stop on runlevel [06] - -# Respawn it if the process exits -respawn -respawn limit 5 30 -expect fork - -# You need to chdir somewhere writable because logstash needs to unpack a few -# temporary files on startup. -chdir /opt/logstash - -script - - # This runs logstash agent as the 'logstash' user - su -s /bin/sh -c 'exec "$0" "$@"' logstash -- /usr/bin/java -jar logstash.jar agent -f /etc/logstash/agent.conf --log /var/log/logstash/agent.log & - emit logstash-agent-running -end script diff --git a/files/logstash-indexer.conf b/files/logstash-indexer.conf deleted file mode 100644 index bd12ea1..0000000 --- a/files/logstash-indexer.conf +++ /dev/null @@ -1,27 +0,0 @@ -# logstash - indexer instance -# -# Adapted from http://cookbook.logstash.net/recipes/using-upstart/ - -description "logstash indexer instance" - -start on (virtual-filesystems and started networking) -stop on runlevel [06] - -# Respawn it if the process exits -respawn -respawn limit 5 30 - -# You need to chdir somewhere writable because logstash needs to unpack a few -# temporary files on startup. -chdir /opt/logstash - -setuid logstash - -script - JAVA_ARGS='' - if [ -f /etc/default/logstash-indexer ] ; - then - . /etc/default/logstash-indexer - fi - exec /usr/bin/java $JAVA_ARGS -jar logstash.jar agent -f /etc/logstash/indexer.conf --log /var/log/logstash/indexer.log -end script diff --git a/files/logstash-web.conf b/files/logstash-web.conf deleted file mode 100644 index 05e16f7..0000000 --- a/files/logstash-web.conf +++ /dev/null @@ -1,24 +0,0 @@ -# logstash - web instance -# -# Copied from http://cookbook.logstash.net/recipes/using-upstart/ - -description "logstash web instance" - -start on virtual-filesystems -stop on runlevel [06] - -# Respawn it if the process exits -respawn -respawn limit 5 30 -expect fork - -# You need to chdir somewhere writable because logstash needs to unpack a few -# temporary files on startup. -chdir /opt/logstash - -script - - # This runs logstash web as the 'logstash' user - su -s /bin/sh -c 'exec "$0" "$@"' logstash -- /usr/bin/java -jar logstash.jar web --backend elasticsearch://127.0.0.1/ --log /var/log/logstash/web.log & - emit logstash-web-running -end script diff --git a/files/logstash.default b/files/logstash.default new file mode 100644 index 0000000..9f1cd9a --- /dev/null +++ b/files/logstash.default @@ -0,0 +1,41 @@ +############################## +# Default settings for logstash +############################### + +# Override Java location +#JAVACMD=/usr/bin/java + +# Set a home directory +#LS_HOME=/var/lib/logstash + +# Arguments to pass to logstash agent +# Single worker so multiline events work +LS_OPTS="-w 1" + +# Arguments to pass to java +LS_HEAP_SIZE="2g" +#LS_JAVA_OPTS="-Djava.io.tmpdir=$HOME" + +# pidfiles aren't used for upstart; this is for sysv users. +#LS_PIDFILE=/var/run/logstash.pid + +# user id to be invoked as; for upstart: edit /etc/init/logstash.conf +#LS_USER=logstash + +# logstash logging +#LS_LOG_FILE=/var/log/logstash/logstash.log +#LS_USE_GC_LOGGING="true" + +# logstash configuration directory +#LS_CONF_DIR=/etc/logstash/conf.d + +# Open file limit; cannot be overridden in upstart +#LS_OPEN_FILES=16384 + +# Nice level +#LS_NICE=19 + +# If this is set to 1, then when `stop` is called, if the process has +# not exited within a reasonable time, SIGKILL will be sent next. +# The default behavior is to simply log a message "program stop failed; still running" +KILL_ON_STOP_TIMEOUT=0 diff --git a/manifests/agent.pp b/manifests/agent.pp index 34d8545..f8d6c6c 100644 --- a/manifests/agent.pp +++ b/manifests/agent.pp @@ -25,32 +25,8 @@ class logstash::agent ( $conf_template = 'logstash/agent.conf.erb' ) { - include ::logstash - - file { '/etc/init/logstash-agent.conf': - ensure => present, - source => 'puppet:///modules/logstash/logstash-agent.conf', - replace => true, - owner => 'root', - } - - file { '/etc/logstash/agent.conf': - ensure => present, - content => template($conf_template), - replace => true, - owner => 'logstash', - group => 'logstash', - mode => '0644', - require => Class['logstash'], - } - - service { 'logstash-agent': - ensure => running, - enable => true, - subscribe => File['/etc/logstash/agent.conf'], - require => [ - Class['logstash'], - File['/etc/init/logstash-agent.conf'], - ] + warning('This class is deprecated and logstash::indexer should be used instead') + class { '::logstash::indexer': + conf_template => $conf_template, } } diff --git a/manifests/indexer.pp b/manifests/indexer.pp index 620365b..205e8ab 100644 --- a/manifests/indexer.pp +++ b/manifests/indexer.pp @@ -20,20 +20,13 @@ # # [*conf_template*] # String. Path to indexer config template. -# Default: 'logstash/agent.conf.erb' +# Default: 'logstash/indexer.conf.erb' class logstash::indexer ( $conf_template = 'logstash/indexer.conf.erb' ) { include ::logstash - file { '/etc/init/logstash-indexer.conf': - ensure => present, - source => 'puppet:///modules/logstash/logstash-indexer.conf', - replace => true, - owner => 'root', - } - - file { '/etc/logstash/indexer.conf': + file { '/etc/logstash/conf.d/indexer.conf': ensure => present, content => template($conf_template), replace => true, @@ -43,27 +36,23 @@ class logstash::indexer ( require => Class['logstash'], } - service { 'logstash-indexer': - ensure => running, - enable => true, - subscribe => File['/etc/logstash/indexer.conf'], - require => [ - Class['logstash'], - File['/etc/init/logstash-indexer.conf'], - ] + file { '/etc/default/logstash': + ensure => present, + source => 'puppet:///modules/logstash/logstash.default', + replace => true, + owner => 'logstash', + group => 'logstash', + mode => '0644', + require => Class['logstash'], } - include ::logrotate - logrotate::file { 'indexer.log': - log => '/var/log/logstash/indexer.log', - options => [ - 'compress', - 'copytruncate', - 'missingok', - 'rotate 7', - 'daily', - 'notifempty', + service { 'logstash': + ensure => running, + enable => true, + subscribe => [ + File['/etc/logstash/conf.d/indexer.conf'], + File['/etc/default/logstash'], ], - require => Service['logstash-indexer'], + require => Class['logstash'], } } diff --git a/manifests/init.pp b/manifests/init.pp index d3d9bca..4398dd2 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -17,64 +17,11 @@ # Class to install common logstash items. # class logstash { - group { 'logstash': - ensure => present, - } - - user { 'logstash': - ensure => present, - comment => 'Logstash User', - home => '/opt/logstash', - gid => 'logstash', - shell => '/bin/bash', - membership => 'minimum', - require => Group['logstash'], - } - - file { '/opt/logstash': - ensure => directory, - owner => 'logstash', - group => 'logstash', - mode => '0644', - require => User['logstash'], - } - - exec { 'get_logstash_jar': - command => 'wget https://download.elasticsearch.org/logstash/logstash/logstash-1.3.3-flatjar.jar -O /opt/logstash/logstash-1.3.3-flatjar.jar', - path => '/bin:/usr/bin', - creates => '/opt/logstash/logstash-1.3.3-flatjar.jar', - require => File['/opt/logstash'], - } - - file { '/opt/logstash/logstash-1.3.3-flatjar.jar': - ensure => present, - owner => 'logstash', - group => 'logstash', - mode => '0644', - require => [ - User['logstash'], - Exec['get_logstash_jar'], - ] - } - - file { '/opt/logstash/logstash.jar': - ensure => link, - target => '/opt/logstash/logstash-1.3.3-flatjar.jar', - require => File['/opt/logstash/logstash-1.3.3-flatjar.jar'], - } - - file { '/var/log/logstash': - ensure => directory, - owner => 'logstash', - group => 'logstash', - mode => '0644', - } - - file { '/etc/logstash': - ensure => directory, - owner => 'logstash', - group => 'logstash', - mode => '0644', + archive { '/tmp/logstash_2.0.0-1_all.deb': + source => 'https://download.elastic.co/logstash/logstash/packages/debian/logstash_2.0.0-1_all.deb', + extract => false, + checksum => '094b18c77d7c959c1203012983337d5249922290', + checksum_type => 'sha1', } if ! defined(Package['openjdk-7-jre-headless']) { @@ -82,4 +29,14 @@ class logstash { ensure => present, } } + + package { 'logstash': + ensure => latest, + source => '/tmp/logstash_2.0.0-1_all.deb', + provider => 'dpkg', + require => [ + Package['openjdk-7-jre-headless'], + Archive['/tmp/logstash_2.0.0-1_all.deb'], + ] + } } diff --git a/manifests/redis.pp b/manifests/redis.pp index 12baa2d..e62de44 100644 --- a/manifests/redis.pp +++ b/manifests/redis.pp @@ -18,6 +18,7 @@ # Class to install redis # class logstash::redis { + warning('This class is deprecated and should not be used') # TODO(clarkb): Access to redis should be controlled at a network level # (with iptables) and with client authentication. Put this in place before # opening redis to external clients. diff --git a/manifests/web.pp b/manifests/web.pp index 9ee5c55..9b8b704 100644 --- a/manifests/web.pp +++ b/manifests/web.pp @@ -57,27 +57,9 @@ class logstash::web ( ensure => present, } - include ::logstash - case $frontend { 'internal': { - file { '/etc/init/logstash-web.conf': - ensure => present, - source => 'puppet:///modules/logstash/logstash-web.conf', - replace => true, - owner => 'root', - } - - service { 'logstash-web': - ensure => running, - enable => true, - require => [ - Class['logstash'], - File['/etc/init/logstash-web.conf'], - ], - } - - $vhost = 'logstash/logstash.vhost.erb' + fail('This stopped working when we moved to Kibana3 and there is no analog in logstash 2.0. Use kibana.') } 'kibana': {