Fix logstash gearman puppet.
* modules/openstack_project/manifests/logstash.pp: Concat is not available in our version of puppetlabs stdlib. Use flatten instead which is available. Remove dependency on non existant logstash::indexer class. Fix requires orders. * modules/openstack_project/manifests/logstash_worker.pp: Fix requires orders. * modules/openstack_project/files/logstash/jenkins-log-client.init * modules/openstack_project/files/logstash/jenkins-log-worker.init: Set pidfile argument when calling scripts. * modules/openstack_project/files/logstash/log-gearman-worker.py: Use python2 compatible gzip.GzipFile instead of gzip.decompress. Send work exception instead of work fail when an exception happens. Log these exceptions locally as well. Change-Id: Idf0a873215acb72187e058a0306a21ccd928d464 Reviewed-on: https://review.openstack.org/32804 Reviewed-by: Jeremy Stanley <fungi@yuggoth.org> Reviewed-by: Khai Do <zaro0508@gmail.com> Reviewed-by: James E. Blair <corvus@inaugust.com> Approved: Clark Boylan <clark.boylan@gmail.com> Tested-by: Jenkins
This commit is contained in:
Clark Boylan
Jenkins
parent
42078ca25a
commit
abe222fa9e
@ -16,8 +16,8 @@ PATH=/sbin:/usr/sbin:/bin:/usr/bin
|
||||
DESC="Jenkins Log Client"
|
||||
NAME=jenkins-log-client
|
||||
DAEMON=/usr/local/bin/log-gearman-client.py
|
||||
DAEMON_ARGS='-c /etc/logstash/jenkins-log-client.yaml -d /var/log/logstash/log-client-debug.log'
|
||||
PIDFILE=/var/run/$NAME/$NAME.pid
|
||||
DAEMON_ARGS="-c /etc/logstash/jenkins-log-client.yaml -d /var/log/logstash/log-client-debug.log -p $PIDFILE"
|
||||
SCRIPTNAME=/etc/init.d/$NAME
|
||||
USER=logstash
|
||||
|
||||
|
||||
@ -16,8 +16,8 @@ PATH=/sbin:/usr/sbin:/bin:/usr/bin
|
||||
DESC="Jenkins Log Worker"
|
||||
NAME=jenkins-log-worker
|
||||
DAEMON=/usr/local/bin/log-gearman-worker.py
|
||||
DAEMON_ARGS='-c /etc/logstash/jenkins-log-worker.yaml -d /var/log/logstash/log-worker-debug.log'
|
||||
PIDFILE=/var/run/$NAME/$NAME.pid
|
||||
DAEMON_ARGS="-c /etc/logstash/jenkins-log-worker.yaml -d /var/log/logstash/log-worker-debug.log -p $PIDFILE"
|
||||
SCRIPTNAME=/etc/init.d/$NAME
|
||||
USER=logstash
|
||||
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
import cStringIO
|
||||
import daemon
|
||||
import gear
|
||||
import gzip
|
||||
@ -83,8 +84,9 @@ class LogRetriever(threading.Thread):
|
||||
out_event["event_message"] = line
|
||||
self.logq.put(out_event)
|
||||
job.sendWorkComplete()
|
||||
except:
|
||||
job.sendWorkFail()
|
||||
except Exception as e:
|
||||
logging.exception("Exception handling log event.")
|
||||
job.sendWorkException(str(e).encode('utf-8'))
|
||||
|
||||
def _retrieve_log(self, source_url, retry):
|
||||
# TODO (clarkb): This should check the content type instead of file
|
||||
@ -100,7 +102,11 @@ class LogRetriever(threading.Thread):
|
||||
logging.exception("Unable to get log data.")
|
||||
if gzipped:
|
||||
logging.debug("Decompressing gzipped source file.")
|
||||
buf = gzip.decompress(raw_buf).decode('utf-8')
|
||||
raw_strIO = cStringIO.StringIO(raw_buf)
|
||||
f = gzip.GzipFile(fileobj=raw_strIO)
|
||||
buf = f.read().decode('utf-8')
|
||||
raw_strIO.close()
|
||||
f.close()
|
||||
else:
|
||||
logging.debug("Decoding source file.")
|
||||
buf = raw_buf.decode('utf-8')
|
||||
|
||||
@ -21,7 +21,7 @@ class openstack_project::logstash (
|
||||
) {
|
||||
$iptables_es_rule = regsubst ($elasticsearch_masters, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT')
|
||||
$iptables_gm_rule = regsubst ($gearman_workers, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 4730 -s \1 -j ACCEPT')
|
||||
$iptables_rule = concat($iptables_es_rule, $iptables_gm_rule)
|
||||
$iptables_rule = flatten([$iptables_es_rule, $iptables_gm_rule])
|
||||
class { 'openstack_project::server':
|
||||
iptables_public_tcp_ports => [22, 80],
|
||||
iptables_rules6 => $iptables_rule,
|
||||
@ -59,6 +59,12 @@ class openstack_project::logstash (
|
||||
group => 'root',
|
||||
mode => '0755',
|
||||
source => 'puppet:///modules/openstack_project/logstash/log-gearman-client.py',
|
||||
require => [
|
||||
Package['python-daemon'],
|
||||
Package['python-zmq'],
|
||||
Package['python-yaml'],
|
||||
Package['gear'],
|
||||
],
|
||||
}
|
||||
|
||||
file { '/etc/logstash/jenkins-log-client.yaml':
|
||||
@ -67,7 +73,6 @@ class openstack_project::logstash (
|
||||
group => 'root',
|
||||
mode => '0555',
|
||||
source => 'puppet:///modules/openstack_project/logstash/jenkins-log-client.yaml',
|
||||
require => Class['logstash::indexer'],
|
||||
}
|
||||
|
||||
file { '/etc/init.d/jenkins-log-client':
|
||||
@ -77,7 +82,7 @@ class openstack_project::logstash (
|
||||
mode => '0555',
|
||||
source => 'puppet:///modules/openstack_project/logstash/jenkins-log-client.init',
|
||||
require => [
|
||||
File['/usr/local/bin/log-client.py'],
|
||||
File['/usr/local/bin/log-gearman-client.py'],
|
||||
File['/etc/logstash/jenkins-log-client.yaml'],
|
||||
],
|
||||
}
|
||||
|
||||
@ -55,6 +55,12 @@ class openstack_project::logstash_worker (
|
||||
group => 'root',
|
||||
mode => '0755',
|
||||
source => 'puppet:///modules/openstack_project/logstash/log-gearman-worker.py',
|
||||
require => [
|
||||
Package['python-daemon'],
|
||||
Package['python-zmq'],
|
||||
Package['python-yaml'],
|
||||
Package['gear'],
|
||||
],
|
||||
}
|
||||
|
||||
file { '/etc/logstash/jenkins-log-worker.yaml':
|
||||
@ -63,7 +69,6 @@ class openstack_project::logstash_worker (
|
||||
group => 'root',
|
||||
mode => '0555',
|
||||
source => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
|
||||
require => Class['logstash::indexer'],
|
||||
}
|
||||
|
||||
file { '/etc/init.d/jenkins-log-worker':
|
||||
@ -73,7 +78,7 @@ class openstack_project::logstash_worker (
|
||||
mode => '0555',
|
||||
source => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.init',
|
||||
require => [
|
||||
File['/usr/local/bin/log-worker.py'],
|
||||
File['/usr/local/bin/log-gearman-worker.py'],
|
||||
File['/etc/logstash/jenkins-log-worker.yaml'],
|
||||
],
|
||||
}
|
||||
@ -82,6 +87,9 @@ class openstack_project::logstash_worker (
|
||||
enable => true,
|
||||
hasrestart => true,
|
||||
subscribe => File['/etc/logstash/jenkins-log-worker.yaml'],
|
||||
require => File['/etc/init.d/jenkins-log-worker'],
|
||||
require => [
|
||||
Class['logstash::indexer'],
|
||||
File['/etc/init.d/jenkins-log-worker'],
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user