Deploy elastic-recheck on logstash.openstack.org.
This elastic recheck thing that classifies failures based on logstash queries is awesome. We should run it on the OpenStack developer infrastructure. Deploy it on logstash.o.o to keep related services together. Change-Id: I16ae7a32c2c48474454df17931779d0f60c3ed85
This commit is contained in:
Clark Boylan
Jeremy Stanley
parent
bd60e3a4a4
commit
e0427cfbb7
@ -257,9 +257,9 @@ $elasticsearch_nodes = [
|
||||
|
||||
node 'logstash.openstack.org' {
|
||||
class { 'openstack_project::logstash':
|
||||
sysadmins => hiera('sysadmins'),
|
||||
elasticsearch_nodes => $elasticsearch_nodes,
|
||||
gearman_workers => [
|
||||
sysadmins => hiera('sysadmins'),
|
||||
elasticsearch_nodes => $elasticsearch_nodes,
|
||||
gearman_workers => [
|
||||
'logstash-worker1.openstack.org',
|
||||
'logstash-worker2.openstack.org',
|
||||
'logstash-worker3.openstack.org',
|
||||
@ -269,7 +269,7 @@ node 'logstash.openstack.org' {
|
||||
'logstash-worker7.openstack.org',
|
||||
'logstash-worker8.openstack.org',
|
||||
],
|
||||
discover_nodes => [
|
||||
discover_nodes => [
|
||||
'elasticsearch.openstack.org:9200',
|
||||
'elasticsearch2.openstack.org:9200',
|
||||
'elasticsearch3.openstack.org:9200',
|
||||
@ -277,6 +277,13 @@ node 'logstash.openstack.org' {
|
||||
'elasticsearch5.openstack.org:9200',
|
||||
'elasticsearch6.openstack.org:9200',
|
||||
],
|
||||
# Config for elastic-recheck
|
||||
gerrit_ssh_private_key => '/etc/elastic-recheck/id_rsa',
|
||||
gerrit_ssh_private_key_contents => hiera('elastic-recheck_gerrit_ssh_private_key'),
|
||||
recheck_bot_nick => 'openstackrecheck',
|
||||
recheck_bot_passwd => hiera('elastic-recheck_ircbot_password'),
|
||||
gerrit_host => 'review.openstack.org',
|
||||
elasticsearch_url => 'http://logstash.openstack.org/elasticsearch/',
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
150
modules/elastic_recheck/files/elastic-recheck.init
Normal file
150
modules/elastic_recheck/files/elastic-recheck.init
Normal file
@ -0,0 +1,150 @@
|
||||
#! /bin/sh
|
||||
### BEGIN INIT INFO
|
||||
# Provides: elastic-recheck
|
||||
# Required-Start: $remote_fs $syslog
|
||||
# Required-Stop: $remote_fs $syslog
|
||||
# Default-Start: 2 3 4 5
|
||||
# Default-Stop: 0 1 6
|
||||
# Short-Description: Elastic Recheck
|
||||
# Description: Classify tempest-devstack failures using ElasticSearch
|
||||
### END INIT INFO
|
||||
|
||||
# Jeremy Stanley <fungi@yuggoth.org>
|
||||
|
||||
# Do NOT "set -e"
|
||||
|
||||
# PATH should only include /usr/* if it runs after the mountnfs.sh script
|
||||
PATH=/sbin:/usr/sbin:/bin:/usr/bin
|
||||
DESC="Elastic Recheck"
|
||||
NAME=elastic-recheck
|
||||
#TODO: rework the following two lines once we have executable entrypoints
|
||||
DAEMON=/usr/bin/python
|
||||
DAEMON_ARGS="/opt/$NAME/elastic_recheck/bot.py /etc/$NAME/$NAME.conf"
|
||||
PIDFILE=/var/run/$NAME/$NAME.pid
|
||||
SCRIPTNAME=/etc/init.d/$NAME
|
||||
USER=recheck
|
||||
|
||||
# Exit if the package is not installed
|
||||
[ -x "$DAEMON" ] || exit 0
|
||||
|
||||
# Read configuration variable file if it is present
|
||||
[ -r /etc/default/$NAME ] && . /etc/default/$NAME
|
||||
|
||||
# Load the VERBOSE setting and other rcS variables
|
||||
. /lib/init/vars.sh
|
||||
|
||||
# Define LSB log_* functions.
|
||||
# Depend on lsb-base (>= 3.0-6) to ensure that this file is present.
|
||||
. /lib/lsb/init-functions
|
||||
|
||||
#
|
||||
# Function that starts the daemon/service
|
||||
#
|
||||
do_start()
|
||||
{
|
||||
# Return
|
||||
# 0 if daemon has been started
|
||||
# 1 if daemon was already running
|
||||
# 2 if daemon could not be started
|
||||
|
||||
mkdir -p /var/run/$NAME
|
||||
chown $USER /var/run/$NAME
|
||||
start-stop-daemon --start --quiet --pidfile $PIDFILE -c $USER --exec $DAEMON --test > /dev/null \
|
||||
|| return 1
|
||||
start-stop-daemon --start --quiet --pidfile $PIDFILE -c $USER --exec $DAEMON -- \
|
||||
$DAEMON_ARGS \
|
||||
|| return 2
|
||||
# Add code here, if necessary, that waits for the process to be ready
|
||||
# to handle requests from services started subsequently which depend
|
||||
# on this one. As a last resort, sleep for some time.
|
||||
}
|
||||
|
||||
#
|
||||
# Function that stops the daemon/service
|
||||
#
|
||||
do_stop()
|
||||
{
|
||||
# Return
|
||||
# 0 if daemon has been stopped
|
||||
# 1 if daemon was already stopped
|
||||
# 2 if daemon could not be stopped
|
||||
# other if a failure occurred
|
||||
start-stop-daemon --stop --signal 9 --pidfile $PIDFILE
|
||||
RETVAL="$?"
|
||||
[ "$RETVAL" = 2 ] && return 2
|
||||
rm -f /var/run/$NAME/*
|
||||
return "$RETVAL"
|
||||
}
|
||||
|
||||
#
|
||||
# Function that sends a SIGHUP to the daemon/service
|
||||
#
|
||||
do_reload() {
|
||||
#
|
||||
# If the daemon can reload its configuration without
|
||||
# restarting (for example, when it is sent a SIGHUP),
|
||||
# then implement that here.
|
||||
#
|
||||
start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name $NAME
|
||||
return 0
|
||||
}
|
||||
|
||||
case "$1" in
|
||||
start)
|
||||
[ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME"
|
||||
do_start
|
||||
case "$?" in
|
||||
0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
|
||||
2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
|
||||
esac
|
||||
;;
|
||||
stop)
|
||||
[ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME"
|
||||
do_stop
|
||||
case "$?" in
|
||||
0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
|
||||
2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
|
||||
esac
|
||||
;;
|
||||
status)
|
||||
status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $?
|
||||
;;
|
||||
#reload|force-reload)
|
||||
#
|
||||
# If do_reload() is not implemented then leave this commented out
|
||||
# and leave 'force-reload' as an alias for 'restart'.
|
||||
#
|
||||
#log_daemon_msg "Reloading $DESC" "$NAME"
|
||||
#do_reload
|
||||
#log_end_msg $?
|
||||
#;;
|
||||
restart|force-reload)
|
||||
#
|
||||
# If the "reload" option is implemented then remove the
|
||||
# 'force-reload' alias
|
||||
#
|
||||
log_daemon_msg "Restarting $DESC" "$NAME"
|
||||
do_stop
|
||||
case "$?" in
|
||||
0|1)
|
||||
do_start
|
||||
case "$?" in
|
||||
0) log_end_msg 0 ;;
|
||||
1) log_end_msg 1 ;; # Old process is still running
|
||||
*) log_end_msg 1 ;; # Failed to start
|
||||
esac
|
||||
;;
|
||||
*)
|
||||
# Failed to stop
|
||||
log_end_msg 1
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
*)
|
||||
#echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2
|
||||
echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
|
||||
exit 3
|
||||
;;
|
||||
esac
|
||||
|
||||
:
|
||||
4
modules/elastic_recheck/files/recheckwatchbot.yaml
Normal file
4
modules/elastic_recheck/files/recheckwatchbot.yaml
Normal file
@ -0,0 +1,4 @@
|
||||
openstack-qa:
|
||||
events:
|
||||
- positive
|
||||
- negative
|
||||
131
modules/elastic_recheck/manifests/init.pp
Normal file
131
modules/elastic_recheck/manifests/init.pp
Normal file
@ -0,0 +1,131 @@
|
||||
# Copyright 2013 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
# Class to install and configure an instance of the elastic-recheck
|
||||
# service.
|
||||
#
|
||||
class elastic_recheck (
|
||||
$gerrit_host,
|
||||
$gerrit_ssh_private_key,
|
||||
$gerrit_ssh_private_key_contents,
|
||||
#not used today, will be used when elastic-recheck supports it.
|
||||
$elasticsearch_url,
|
||||
$recheck_bot_passwd,
|
||||
$gerrit_user = 'elasticrecheck',
|
||||
$recheck_bot_nick = 'openstackrecheck'
|
||||
) {
|
||||
group { 'recheck':
|
||||
ensure => 'present',
|
||||
}
|
||||
|
||||
user { 'recheck':
|
||||
ensure => present,
|
||||
home => '/var/run/elastic-recheck',
|
||||
shell => '/bin/false',
|
||||
gid => 'recheck',
|
||||
require => [
|
||||
Group['recheck'],
|
||||
File['/var/run/elastic-recheck'],
|
||||
],
|
||||
}
|
||||
|
||||
vcsrepo { '/opt/elastic-recheck':
|
||||
ensure => latest,
|
||||
provider => git,
|
||||
revision => 'master',
|
||||
source => 'https://git.openstack.org/openstack-infra/elastic-recheck',
|
||||
}
|
||||
|
||||
include pip
|
||||
exec { 'install_elastic-recheck' :
|
||||
command => 'python setup.py install',
|
||||
cwd => '/opt/elastic-recheck',
|
||||
path => '/bin:/usr/bin',
|
||||
refreshonly => true,
|
||||
subscribe => Vcsrepo['/opt/elastic-recheck'],
|
||||
require => Class['pip'],
|
||||
}
|
||||
|
||||
file { '/var/run/elastic-recheck':
|
||||
ensure => directory,
|
||||
mode => '0755',
|
||||
owner => 'recheck',
|
||||
group => 'recheck',
|
||||
}
|
||||
|
||||
file { '/var/log/elastic-recheck':
|
||||
ensure => directory,
|
||||
mode => '0755',
|
||||
owner => 'recheck',
|
||||
group => 'recheck',
|
||||
}
|
||||
|
||||
file { '/etc/elastic-recheck':
|
||||
ensure => directory,
|
||||
mode => '0755',
|
||||
owner => 'recheck',
|
||||
group => 'recheck',
|
||||
}
|
||||
|
||||
file { '/etc/elastic-recheck/elastic-recheck.conf':
|
||||
ensure => present,
|
||||
mode => '0640',
|
||||
owner => 'recheck',
|
||||
group => 'recheck',
|
||||
content => template('elastic_recheck/elastic-recheck.conf.erb'),
|
||||
}
|
||||
|
||||
file { '/etc/elastic-recheck/recheckwatchbot.yaml':
|
||||
ensure => present,
|
||||
mode => '0640',
|
||||
owner => 'recheck',
|
||||
group => 'recheck',
|
||||
source => 'puppet:///modules/elastic_recheck/recheckwatchbot.yaml',
|
||||
}
|
||||
|
||||
# TODO(clarkb) put queries.json somewhere else.
|
||||
file { '/etc/elastic-recheck/queries.json':
|
||||
ensure => link,
|
||||
target => '/opt/elastic-recheck/queries.json',
|
||||
require => Vcsrepo['/opt/elastic-recheck'],
|
||||
}
|
||||
|
||||
file { $gerrit_ssh_private_key:
|
||||
ensure => present,
|
||||
mode => '0600',
|
||||
owner => 'recheck',
|
||||
group => 'recheck',
|
||||
content => $gerrit_ssh_private_key_contents,
|
||||
}
|
||||
|
||||
file { '/etc/init.d/elastic-recheck':
|
||||
ensure => present,
|
||||
mode => '0755',
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
source => 'puppet:///modules/elastic_recheck/elastic-recheck.init',
|
||||
}
|
||||
|
||||
service { 'elastic-recheck':
|
||||
ensure => running,
|
||||
enable => true,
|
||||
subscribe => File['/etc/elastic-recheck/elastic-recheck.conf'],
|
||||
require => [
|
||||
File['/etc/init.d/elastic-recheck'],
|
||||
File['/etc/elastic-recheck/elastic-recheck.conf'],
|
||||
File['/etc/elastic-recheck/queries.json'],
|
||||
Exec['install_elastic-recheck'],
|
||||
],
|
||||
}
|
||||
}
|
||||
13
modules/elastic_recheck/templates/elastic-recheck.conf.erb
Normal file
13
modules/elastic_recheck/templates/elastic-recheck.conf.erb
Normal file
@ -0,0 +1,13 @@
|
||||
#Must use full paths
|
||||
[ircbot]
|
||||
nick=<%= recheck_bot_nick %>
|
||||
pass=<%= recheck_bot_passwd %>
|
||||
server=irc.freenode.net
|
||||
port=6667
|
||||
channel_config=/etc/elastic-recheck/recheckwatchbot.yaml
|
||||
|
||||
[gerrit]
|
||||
user=<%= gerrit_user %>
|
||||
host=<%= gerrit_host %>
|
||||
query_file=/etc/elastic-recheck/queries.json
|
||||
key=<%= gerrit_ssh_private_key %>
|
||||
@ -15,6 +15,13 @@
|
||||
# Logstash web frontend glue class.
|
||||
#
|
||||
class openstack_project::logstash (
|
||||
$gerrit_host,
|
||||
$gerrit_ssh_private_key,
|
||||
$gerrit_ssh_private_key_contents,
|
||||
#not used today, will be used when elastic-recheck supports it.
|
||||
$elasticsearch_url,
|
||||
$recheck_bot_passwd,
|
||||
$recheck_bot_nick = 'openstackrecheck',
|
||||
$elasticsearch_nodes = [],
|
||||
$gearman_workers = [],
|
||||
$discover_nodes = ['elasticsearch.openstack.org:9200'],
|
||||
@ -109,4 +116,13 @@ class openstack_project::logstash (
|
||||
],
|
||||
require => Service['jenkins-log-client'],
|
||||
}
|
||||
|
||||
class { 'elastic_recheck':
|
||||
gerrit_host => $gerrit_host,
|
||||
gerrit_ssh_private_key => $gerrit_ssh_private_key,
|
||||
gerrit_ssh_private_key_contents => $gerrit_ssh_private_key_contents,
|
||||
elasticsearch_url => $elasticsearch_url,
|
||||
recheck_bot_passwd => $recheck_bot_passwd,
|
||||
recheck_bot_nick => $recheck_bot_nick,
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user