#!/usr/bin/python
import copy
import datetime
import json
import sys
import time
from collections import namedtuple
import conf
def css():
with open('rcbau.css', 'w') as f:
f.write("""
body {
font-size:75%;
color:#222;
background:#fff;
font-family:"Helvetica Neue", Arial, Helvetica, sans-serif;
}
h3.failing { color: red; }
h3.nonvoting { color: blue; }
div.graph { float:left; width:25em; height:300px; }
div.small_graph { width:15em; height:150px; }
div.patchsets_list { width:50em; float:left }
div.clear_float { clear:both }
a.ui-tabs-anchor div {
color: blue;
text-align: center;
text-decoration: underline;
}
""")
def patch_list_as_html(l):
out = []
for p in sorted(l):
number, patch = p.split(',')
out.append('%s,%s'
% (number, patch, number, patch))
return ', '.join(out)
def pie_chart(author):
name = author.name.replace(' ', '-')
passed = author.passed
failed = author.failed
missed = len(author.missed_votes)
return """
$('#{name}-graph').highcharts({{
chart: {{
plotBackgroundColor: null,
plotBorderWidth: 0,
plotShadow: false
}},
colors: [ 'green', 'red', 'yellow' ],
title: {{
text: 'Results',
align: 'center',
verticalAlign: 'middle',
y: 50
}},
tooltip: {{
pointFormat:
'{{series.name}}: {{point.percentage:.1f}}%'
}},
plotOptions: {{
pie: {{
dataLabels: {{
enabled: true,
distance: -50,
style: {{
fontWeight: 'bold',
color: 'white',
textShadow: '0px 1px 2px black'
}}
}},
startAngle: -90,
endAngle: 90,
center: ['50%', '75%']
}}
}},
series: [{{
type: 'pie',
name: 'Results',
innerSize: '50%',
data: [
['Passed', {passed}],
['Failed', {failed}],
['Missed', {missed}]
]
}}]
}});
""".format(name=name, passed=passed, failed=failed, missed=missed)
def small_pie_chart(author):
name = author.name.replace(' ', '-') + '-small'
if author.total:
passed = author.passed
failed = author.failed
missed = len(author.missed_votes)
else:
passed = 0
failed = 0
missed = 1
graph = """
$('#{name}-graph').highcharts({{
chart: {{
plotBackgroundColor: null,
plotBorderWidth: 0,
plotShadow: false
}},
colors: [ 'green', 'red', 'yellow' ],
title: {{
text: '',
align: 'center',
verticalAlign: 'middle',
y: 50
}},
tooltip: {{
pointFormat:
'{{series.name}}: {{point.percentage:.1f}}%'
}},
plotOptions: {{
pie: {{
dataLabels: {{
enabled: false
}},
startAngle: -90,
endAngle: 90,
center: ['50%', '75%']
}}
}},
series: [{{
type: 'pie',
innerSize: '50%',
data: [
['Passed', {passed}],
['Failed', {failed}],
['Missed', {missed}]
]
}}]
}});
"""
return graph.format(
author=author,
name=name,
passed=passed,
failed=failed,
missed=missed
)
def templated_report(**kwargs):
import os
path = os.path.dirname(__file__)
from chameleon import PageTemplateLoader
templates = PageTemplateLoader(os.path.join(path, "templates"))
template = templates['report.pt']
return template(**kwargs)
def report(project_filter, user_filter, prefix, fname):
with open(fname) as f:
patchsets = json.loads(f.read())
if not user_filter:
user_filter = conf.CI_SYSTEM[prefix]
elif user_filter and not 'Jenkins' in user_filter:
user_filter_new = ['Jenkins']
user_filter_new.extend(user_filter)
user_filter = user_filter_new
user_filter_without_jenkins = copy.copy(user_filter)
user_filter_without_jenkins.remove('Jenkins')
# This is more complicated than it looks because we need to handle
# patchsets which are uploaded so rapidly that older patchsets aren't
# finished testing.
total_patches = 0
total_votes = {}
missed_votes = {}
sentiments = {}
passed_votes = {}
failed_votes = {}
unparsed_votes = {}
for number in patchsets:
if patchsets[number].get('__exemption__'):
continue
if project_filter != '*':
if patchsets[number].get('__project__') != project_filter:
continue
patches = sorted(patchsets[number].keys())
valid_patches = []
# Determine how long a patch was valid for. If it wasn't valid for
# at least three hours, disgard.
for patch in patches:
if not '__created__' in patchsets[number][patch]:
continue
uploaded = datetime.datetime.fromtimestamp(
patchsets[number][patch]['__created__']
)
obsoleted = datetime.datetime.fromtimestamp(
patchsets[number].get(
str(
int(patch) + 1), {}
).get(
'__created__',
time.time()
)
)
valid_for = obsoleted - uploaded
if valid_for < datetime.timedelta(hours=3):
continue
matched_authors = 0
for author in patchsets[number][patch]:
if author == '__created__':
continue
if author not in user_filter_without_jenkins:
continue
matched_authors += 1
if matched_authors == 0:
continue
valid_patches.append(patch)
total_patches += len(valid_patches)
for patch in valid_patches:
for author in patchsets[number][patch]:
if author == '__created__':
continue
if author not in user_filter:
continue
total_votes.setdefault(author, 0)
total_votes[author] += 1
for vote, msg, sentiment in patchsets[number][patch][author]:
if sentiment.startswith('Positive'):
passed_votes.setdefault(author, 0)
passed_votes[author] += 1
elif sentiment.startswith('Negative'):
failed_votes.setdefault(author, 0)
failed_votes[author] += 1
else:
unparsed_votes.setdefault(author, 0)
unparsed_votes[author] += 1
sentiments.setdefault(author, {})
sentiments[author].setdefault(sentiment, [])
sentiments[author][sentiment].append(
'%s,%s' % (number, patch))
for author in user_filter:
if not author in patchsets[number][patch]:
missed_votes.setdefault(author, [])
missed_votes[author].append('%s,%s' % (number, patch))
charts = []
small_charts = []
authors = []
Author = namedtuple(
'Author',
[
'name',
'percentage',
'passed',
'failed',
'unparsed',
'total',
'pass_percentage',
'fail_percentage',
'unparsed_percentage',
'missed_votes',
'sentiments'
]
)
Patch = namedtuple(
'Patch',
'number patch'
)
for user in user_filter:
passed = passed_votes.get(user, 0)
failed = failed_votes.get(user, 0)
unparsed = unparsed_votes.get(user, 0)
total = passed + failed + unparsed
user_sentiments = sentiments.get(user, {})
for sentiment in user_sentiments:
patches = []
for patch in user_sentiments[sentiment]:
patch, number = patch.split(',')
patches.append(Patch(patch, number))
user_sentiments[sentiment] = patches
missed = []
for missed_vote in missed_votes.get(user, []):
patch, number = missed_vote.split(',')
missed.append(Patch(patch, number))
if user in total_votes:
authors.append(
Author(
name=user,
percentage=round(
total_votes[user] * 100.0 / total_patches, 2
),
passed=passed,
failed=failed,
unparsed=unparsed,
total=total,
pass_percentage=round(passed * 100.0 / total, 2),
fail_percentage=round(failed * 100.0 / total, 2),
unparsed_percentage=round(unparsed * 100.0 / total, 2),
missed_votes=missed,
sentiments=user_sentiments
)
)
charts.append(pie_chart(authors[-1]))
small_charts.append(small_pie_chart(authors[-1]))
else:
authors.append(
Author(
name=user,
percentage=None,
passed=None,
failed=None,
unparsed=None,
total=None,
pass_percentage=None,
fail_percentage=None,
unparsed_percentage=None,
missed_votes=None,
sentiments=None
)
)
small_charts.append(small_pie_chart(authors[-1]))
report = templated_report(
total_patches=total_patches,
authors=authors,
now=datetime.datetime.now(),
missed_votes=missed_votes,
sentiments=conf.SENTIMENTS,
total_votes=total_votes,
charts=charts,
small_charts=small_charts
)
with open('%s-cireport.html' % prefix, 'w') as f:
f.write(report)
if __name__ == '__main__':
if len(sys.argv) > 1:
fname = sys.argv[1]
else:
fname = 'patchsets.json'
css()
report('openstack/nova', None, 'nova', fname)
report('openstack/neutron', None, 'neutron', fname)
for user in conf.CI_USERS:
report('*', [user], user.replace(' ', '_'), fname)