Add recheckwatch.

* New puppet class to install the script, init script, and set up
  user and directories.
* Add class to zuul server.
* Install openstack themed scoreboard template.
* Add URL to zuul.openstack.org/rechecks.html.

Change-Id: I9046cd21923aae40107f0d558080c44f65481fd7
Reviewed-on: https://review.openstack.org/18442
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Approved: Jeremy Stanley <fungi@yuggoth.org>
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Tested-by: Jenkins
This commit is contained in:
James E. Blair 2012-12-19 12:28:17 -08:00 committed by Jenkins
parent 1875a89e4b
commit 6bf249ee4b
8 changed files with 604 additions and 14 deletions

View File

@ -0,0 +1,124 @@
<html xmlns="http://www.w3.org/1999/xhtml"
xmlns:py="http://genshi.edgewall.org/"
lang="en">
<HEAD>
<TITLE></TITLE>
<!-- Google Fonts -->
<link href='http://fonts.googleapis.com/css?family=PT+Sans&amp;subset=latin' rel='stylesheet' type='text/css'/>
<!-- Framework CSS -->
<link rel="stylesheet" href="http://openstack.org/themes/openstack/css/blueprint/screen.css" type="text/css" media="screen, projection"/>
<link rel="stylesheet" href="http://openstack.org/themes/openstack/css/blueprint/print.css" type="text/css" media="print"/>
<!-- IE CSS -->
<!--[if lt IE 8]><link rel="stylesheet" href="http://openstack.org/blueprint/ie.css" type="text/css" media="screen, projection"><![endif]-->
<!-- OpenStack Specific CSS -->
<link rel="stylesheet" href="http://openstack.org/themes/openstack/css/dropdown.css" type="text/css" media="screen, projection, print"/>
<!-- Page Specific CSS -->
<link rel="stylesheet" href="http://openstack.org/themes/openstack/css/home.css" type="text/css" media="screen, projection, print"/>
<link rel="stylesheet" type="text/css" href="http://openstack.org/themes/openstack/css/main.css" />
</HEAD>
<BODY>
<div class="container">
<div id="header">
<div class="span-5">
<h1 id="logo"><a href="/">Open Stack</a></h1>
</div>
<div class="span-19 last blueLine">
<div id="navigation" class="span-19">
<ul id="Menu1">
<li><a href="http://openstack.org/" title="Go to the Home page" class="link" >Home</a></li>
<li><a href="http://openstack.org/projects/" title="Go to the OpenStack Projects page" class="link">Projects</a></li>
<li><a href="http://openstack.org/user-stories/" title="Go to the User Stories page" class="link">User Stories</a></li>
<li><a href="http://openstack.org/community/" title="Go to the Community page" class="current">Community</a></li>
<li><a href="http://openstack.org/blog/" title="Go to the OpenStack Blog">Blog</a></li>
<li><a href="http://wiki.openstack.org/" title="Go to the OpenStack Wiki">Wiki</a></li>
<li><a href="http://docs.openstack.org/" title="Go to OpenStack Documentation">Documentation</a></li>
</ul>
</div>
</div>
</div>
</div>
<div class="container">
<h1>Bugs Causing Rechecks</h1>
<br/>
<div py:for="bug in bugs">
<h3 class="subhead">
<a href="https://code.launchpad.net/bugs/${bug['number']}">
Bug ${bug.number}: ${bug.title}</a>
</h3>
First seen: ${bug.first_seen.strftime("%Y-%m-%d %H:%M:%S")} UTC<br/>
Last seen: ${bug.last_seen.strftime("%Y-%m-%d %H:%M:%S")} UTC<br/>
Rechecks: ${len(bug.hits)}<br/>
Affecting projects:
<ul>
<li py:for="project in bug.projects">
${project}
</li>
</ul>
Affecting changes:
<ul>
<li py:for="change in bug.changes">
<a href="https://review-dev.openstack.org/${change}"> ${change} </a>
</li>
</ul>
</div>
</div>
<div class="container">
<hr />
<div id="footer">
<div class="span-4">
<h3>OpenStack</h3>
<ul>
<li><a href="http://openstack.org/projects/">Projects</a></li>
<li><a href="http://openstack.org/openstack-security/">OpenStack Security</a></li>
<li><a href="http://openstack.org/projects/openstack-faq/">Common Questions</a></li>
<li><a href="http://openstack.org/blog/">Blog</a></li>
</ul>
</div>
<div class="span-4">
<h3>Community</h3>
<ul>
<li><a href="http://openstack.org/community/">User Groups</a></li>
<li><a href="http://openstack.org/events/">Events</a></li>
<li><a href="http://openstack.org/jobs/">Jobs</a></li>
<li><a href="http://openstack.org/companies/">Companies</a></li>
<li><a href="http://wiki.openstack.org/HowToContribute">Contribute</a></li>
</ul>
</div>
<div class="span-4">
<h3>Documentation</h3>
<ul>
<li><a href="http://docs.openstack.org/">OpenStack Manuals</a></li>
<li><a href="http://docs.openstack.org/diablo/openstack-compute/starter/content/">Getting Started</a></li>
<li><a href="http://wiki.openstack.org/">Wiki</a></li>
</ul>
</div>
<div class="span-4 last">
<h3>Branding &amp; Legal</h3>
<ul>
<li><a href="http://openstack.org/brand/">Logos &amp; Guidelines</a></li>
<li><a href="http://openstack.org/brand/openstack-trademark-policy/">Trademark Policy</a></li>
<li><a href="http://openstack.org/privacy/">Privacy Policy</a></li>
<li><a href="http://wiki.openstack.org/CLA">OpenStack CLA</a></li>
</ul>
</div>
</div>
</div>
</BODY>
</html>

View File

@ -49,4 +49,16 @@ class openstack_project::zuul(
source => 'puppet:///modules/openstack_project/zuul/logging.conf', source => 'puppet:///modules/openstack_project/zuul/logging.conf',
notify => Exec['zuul-reload'], notify => Exec['zuul-reload'],
} }
class { '::recheckwatch':
gerrit_server => $gerrit_server,
gerrit_user => $gerrit_user,
recheckwatch_ssh_private_key => $zuul_ssh_private_key,
}
file { '/var/lib/recheckwatch/scoreboard.html':
ensure => present,
source => 'puppet:///modules/openstack_project/zuul/scoreboard.html',
require => File['/var/lib/recheckwatch'],
}
} }

View File

@ -0,0 +1,171 @@
#!/usr/bin/env python
# Copyright 2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import ConfigParser
import datetime
import re
import sys
import threading
import traceback
import cPickle as pickle
import os
from genshi.template import TemplateLoader
from launchpadlib.launchpad import Launchpad
from launchpadlib.uris import LPNET_SERVICE_ROOT
import daemon
try:
import daemon.pidlockfile
pid_file_module = daemon.pidlockfile
except:
# as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
# instead it depends on lockfile-0.9.1
import daemon.pidfile
pid_file_module = daemon.pidfile
class Hit(object):
def __init__(self, project, change):
self.project = project
self.change = change
self.ts = datetime.datetime.utcnow()
class Bug(object):
def __init__(self, number):
self.number = number
self.hits = []
self.projects = []
self.changes = []
self.last_seen = None
self.first_seen = None
launchpad = Launchpad.login_anonymously('recheckwatch',
'production')
self.title = launchpad.bugs[number].title
def addHit(self, hit):
self.hits.append(hit)
if not self.first_seen:
self.first_seen = hit.ts
if hit.project not in self.projects:
self.projects.append(hit.project)
if hit.change not in self.changes:
self.changes.append(hit.change)
self.last_seen = hit.ts
class Scoreboard(threading.Thread):
def __init__(self, config):
threading.Thread.__init__(self)
self.scores = {}
server = config.get('gerrit', 'host')
username = config.get('gerrit', 'user')
port = config.getint('gerrit', 'port')
keyfile = config.get('gerrit', 'key', None)
self.pickle_dir = config.get('recheckwatch', 'pickle_dir')
self.pickle_file = os.path.join(self.pickle_dir, 'scoreboard.pickle')
self.template_dir = config.get('recheckwatch', 'template_dir')
self.output_file = config.get('recheckwatch', 'output_file')
self.age = config.getint('recheckwatch', 'age')
self.regex = re.compile(config.get('recheckwatch', 'regex'))
if os.path.exists(self.pickle_file):
out = open(self.pickle_file, 'rb')
self.scores = pickle.load(out)
out.close()
# Import here because it needs to happen after daemonization
import gerritlib.gerrit
self.gerrit = gerritlib.gerrit.Gerrit(server, username, port, keyfile)
self.update()
def _read(self, data):
if data.get('type', '') != 'comment-added':
return
comment = data.get('comment', '')
m = self.regex.match(comment.strip())
if not m:
return
change_record = data.get('change', {})
change = change_record.get('number')
project = change_record.get('project')
bugno = int(m.group('bugno'))
hit = Hit(project, change)
bug = self.scores.get(bugno)
if not bug:
bug = Bug(bugno)
bug.addHit(hit)
self.scores[bugno] = bug
self.update()
def update(self):
# Remove bugs that haven't been seen in ages
to_remove = []
now = datetime.datetime.utcnow()
for bugno, bug in self.scores.items():
if bug.last_seen < now-datetime.timedelta(days=self.age):
to_remove.append(bugno)
for bugno in to_remove:
del self.scores[bug]
def impact(bug):
"This ranks more recent bugs higher"
return (len(bug.hits) * (5.0 / ((bug.last_seen-now).days)))
# Get the bugs reverse sorted by impact
bugs = self.scores.values()
bugs.sort(lambda a,b: cmp(impact(a), impact(b)))
bugs.reverse()
loader = TemplateLoader([self.template_dir], auto_reload=True)
tmpl = loader.load('scoreboard.html')
out = open(self.output_file, 'w')
out.write(tmpl.generate(bugs = bugs).render('html', doctype='html'))
out = open(self.pickle_file, 'wb')
pickle.dump(self.scores, out, -1)
out.close()
def run(self):
self.gerrit.startWatching()
while True:
event = self.gerrit.getEvent()
try:
self._read(event)
except:
traceback.print_exc()
def _main():
config = ConfigParser.ConfigParser()
config.read(sys.argv[1])
s = Scoreboard(config)
s.start()
def main():
if len(sys.argv) < 2:
print "Usage: %s CONFIGFILE" % sys.argv[0]
sys.exit(1)
if '-d' not in sys.argv:
pid = pid_file_module.TimeoutPIDLockFile(
"/var/run/recheckwatch/recheckwatch.pid", 10)
with daemon.DaemonContext(pidfile=pid):
_main()
else:
_main()
if __name__ == "__main__":
main()

View File

@ -0,0 +1,146 @@
#! /bin/sh
### BEGIN INIT INFO
# Provides: recheckwatch
# Required-Start: $remote_fs $syslog
# Required-Stop: $remote_fs $syslog
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: Recheckwatch
# Description: Trunk gating system
### END INIT INFO
# Do NOT "set -e"
# PATH should only include /usr/* if it runs after the mountnfs.sh script
PATH=/sbin:/usr/sbin:/bin:/usr/bin
DESC="Recheckwatch"
NAME=recheckwatch
DAEMON=/usr/local/bin/recheckwatch
DAEMON_ARGS=/etc/recheckwatch/recheckwatch.conf
PIDFILE=/var/run/$NAME/$NAME.pid
SCRIPTNAME=/etc/init.d/$NAME
USER=recheckwatch
# Exit if the package is not installed
[ -x "$DAEMON" ] || exit 0
# Read configuration variable file if it is present
[ -r /etc/default/$NAME ] && . /etc/default/$NAME
# Load the VERBOSE setting and other rcS variables
. /lib/init/vars.sh
# Define LSB log_* functions.
# Depend on lsb-base (>= 3.0-6) to ensure that this file is present.
. /lib/lsb/init-functions
#
# Function that starts the daemon/service
#
do_start()
{
# Return
# 0 if daemon has been started
# 1 if daemon was already running
# 2 if daemon could not be started
mkdir -p /var/run/$NAME
chown $USER /var/run/$NAME
start-stop-daemon --start --quiet --pidfile $PIDFILE -c $USER --exec $DAEMON --test > /dev/null \
|| return 1
start-stop-daemon --start --quiet --pidfile $PIDFILE -c $USER --exec $DAEMON -- \
$DAEMON_ARGS \
|| return 2
# Add code here, if necessary, that waits for the process to be ready
# to handle requests from services started subsequently which depend
# on this one. As a last resort, sleep for some time.
}
#
# Function that stops the daemon/service
#
do_stop()
{
# Return
# 0 if daemon has been stopped
# 1 if daemon was already stopped
# 2 if daemon could not be stopped
# other if a failure occurred
start-stop-daemon --stop --signal 9 --pidfile $PIDFILE
RETVAL="$?"
[ "$RETVAL" = 2 ] && return 2
rm -f /var/run/$NAME/*
return "$RETVAL"
}
#
# Function that stops the daemon/service
#
do_graceful_stop()
{
PID=`cat $PIDFILE`
kill -USR1 $PID
# wait until really stopped
if [ -n "${PID:-}" ]; then
i=0
while kill -0 "${PID:-}" 2> /dev/null; do
if [ $i -eq '0' ]; then
echo -n " ... waiting "
else
echo -n "."
fi
i=$(($i+1))
sleep 1
done
fi
rm -f /var/run/$NAME/*
}
case "$1" in
start)
[ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME"
do_start
case "$?" in
0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
esac
;;
stop)
[ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME"
do_stop
case "$?" in
0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
esac
;;
status)
status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $?
;;
#reload|force-reload)
#
# If do_reload() is not implemented then leave this commented out
# and leave 'force-reload' as an alias for 'restart'.
#
#log_daemon_msg "Reloading $DESC" "$NAME"
#do_reload
#log_end_msg $?
#;;
restart|force-reload)
#
# If the "reload" option is implemented then remove the
# 'force-reload' alias
#
log_daemon_msg "Restarting $DESC" "$NAME"
do_graceful_stop
do_start
;;
*)
#echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2
echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
exit 3
;;
esac
:

View File

@ -0,0 +1,118 @@
# == Class: recheckwatch
#
class recheckwatch (
$gerrit_server = '',
$gerrit_user = '',
$recheckwatch_ssh_private_key = '',
) {
if ! defined(Package['python-daemon']) {
package { 'python-daemon':
ensure => present,
}
}
if ! defined(Package['python-genshi']) {
package { 'python-genshi':
ensure => present,
}
}
if ! defined(Package['python-launchpadlib']) {
package { 'python-launchpadlib':
ensure => present,
}
}
if ! defined(Package['gerritlib']) {
package { 'gerritlib':
ensure => latest,
provider => pip,
require => Class['pip'],
}
}
user { 'recheckwatch':
ensure => present,
home => '/home/recheckwatch',
shell => '/bin/bash',
gid => 'recheckwatch',
managehome => true,
require => Group['recheckwatch'],
}
group { 'recheckwatch':
ensure => present,
}
file { '/etc/recheckwatch':
ensure => directory,
}
file { '/etc/recheckwatch/recheckwatch.conf':
ensure => present,
owner => 'recheckwatch',
mode => '0400',
content => template('recheckwatch/recheckwatch.conf.erb'),
require => [
File['/etc/recheckwatch'],
User['recheckwatch'],
],
}
file { '/var/run/recheckwatch':
ensure => directory,
owner => 'recheckwatch',
require => User['recheckwatch'],
}
file { '/var/www/recheckwatch':
ensure => directory,
owner => 'recheckwatch',
mode => '0755',
require => User['recheckwatch'],
}
file { '/var/lib/recheckwatch':
ensure => directory,
owner => 'recheckwatch',
require => User['recheckwatch'],
}
file { '/var/lib/recheckwatch/ssh':
ensure => directory,
owner => 'recheckwatch',
group => 'recheckwatch',
mode => '0500',
require => File['/var/lib/recheckwatch'],
}
file { '/var/lib/recheckwatch/ssh/id_rsa':
owner => 'recheckwatch',
group => 'recheckwatch',
mode => '0400',
require => File['/var/lib/recheckwatch/ssh'],
content => $recheckwatch_ssh_private_key,
}
file { '/etc/init.d/recheckwatch':
ensure => present,
owner => 'root',
group => 'root',
mode => '0555',
source => 'puppet:///modules/recheckwatch/recheckwatch.init',
}
service { 'recheckwatch':
name => 'recheckwatch',
enable => true,
hasrestart => true,
require => File['/etc/init.d/recheckwatch'],
}
file { '/usr/local/bin/recheckwatch':
ensure => present,
mode => '0555',
source => 'puppet:///modules/recheckwatch/recheckwatch',
}
}

View File

@ -0,0 +1,12 @@
[recheckwatch]
pickle_dir=/var/lib/recheckwatch
template_dir=/var/lib/recheckwatch
output_file=/var/www/recheckwatch/rechecks.html
age=30 ; days
regex=(?i)^(?P<verb>recheck|reverify) (?:bug|lp)[\s#:]*(?P<bugno>\d+)$
[gerrit]
server=<%= gerrit_server %>
user=<%= gerrit_user %>
sshkey=/var/lib/recheckwatch/ssh/id_rsa
port=29418

View File

@ -18,7 +18,6 @@ class zuul (
$packages = [ $packages = [
'python-webob', 'python-webob',
'python-daemon',
'python-lockfile', 'python-lockfile',
'python-paste', 'python-paste',
] ]
@ -27,19 +26,6 @@ class zuul (
ensure => present, ensure => present,
} }
user { 'zuul':
ensure => present,
home => '/home/zuul',
shell => '/bin/bash',
gid => 'zuul',
managehome => true,
require => Group['zuul'],
}
group { 'zuul':
ensure => present,
}
# A lot of things need yaml, be conservative requiring this package to avoid # A lot of things need yaml, be conservative requiring this package to avoid
# conflicts with other modules. # conflicts with other modules.
if ! defined(Package['python-yaml']) { if ! defined(Package['python-yaml']) {
@ -54,6 +40,25 @@ class zuul (
} }
} }
if ! defined(Package['python-daemon']) {
package { 'python-daemon':
ensure => present,
}
}
user { 'zuul':
ensure => present,
home => '/home/zuul',
shell => '/bin/bash',
gid => 'zuul',
managehome => true,
require => Group['zuul'],
}
group { 'zuul':
ensure => present,
}
# Packages that need to be installed from pip # Packages that need to be installed from pip
$pip_packages = [ $pip_packages = [
'GitPython', 'GitPython',

View File

@ -15,6 +15,8 @@
SetEnv GIT_PROJECT_ROOT /var/lib/zuul/git/ SetEnv GIT_PROJECT_ROOT /var/lib/zuul/git/
SetEnv GIT_HTTP_EXPORT_ALL SetEnv GIT_HTTP_EXPORT_ALL
Alias /rechecks.html /var/www/recheckwatch/rechecks.html
AliasMatch ^/p/(.*/objects/[0-9a-f]{2}/[0-9a-f]{38})$ /var/lib/zuul/git/$1 AliasMatch ^/p/(.*/objects/[0-9a-f]{2}/[0-9a-f]{38})$ /var/lib/zuul/git/$1
AliasMatch ^/p/(.*/objects/pack/pack-[0-9a-f]{40}.(pack|idx))$ /var/lib/zuul/git/$1 AliasMatch ^/p/(.*/objects/pack/pack-[0-9a-f]{40}.(pack|idx))$ /var/lib/zuul/git/$1
ScriptAlias /p/ /usr/lib/git-core/git-http-backend/ ScriptAlias /p/ /usr/lib/git-core/git-http-backend/