x/stacktach
Code Issues Proposed changes

Stored report support.

Browse Source 
The "pretty" report can now be stored in the stacktach database
and retrieved via REST or stacky.

Also better command line support for the "pretty" report.
This commit is contained in:
Sandy Walsh 2013-02-18 19:54:24 -04:00 committed by Sandy Walsh
parent 1968fe0899
commit 174348ed48
6 changed files with 96 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
migrations/005_reports.txt Normal file
View File

@ -0,0 +1,3 @@
do
python manage.py syncdb
to add the JsonReport table

99
reports/pretty.py
View File

@ -7,23 +7,24 @@ import time
import prettytable
sys.path.append("/stacktach")
sys.path.append(".")
from stacktach import datetime_to_decimal as dt
from stacktach import image_type
from stacktach import models
def make_report(yesterday=None, start_hour=0, hours=24, percentile=90, store=False):
def make_report(yesterday=None, start_hour=0, hours=24, percentile=90,
store=False):
if not yesterday:
yesterday = datetime.datetime.utcnow().date() - datetime.timedelta(days=1)
yesterday = datetime.datetime.utcnow().date() - \
datetime.timedelta(days=1)
start = datetime.datetime(year=yesterday.year, month=yesterday.month,
day=yesterday.day, hour=start_hour)
end = start + datetime.timedelta(hours=hours-1, minutes=59, seconds=59)
rstart = datetime.datetime(year=yesterday.year, month=yesterday.month,
day=yesterday.day, hour=start_hour)
rend = rstart + datetime.timedelta(hours=hours-1, minutes=59, seconds=59)
dstart = dt.dt_to_decimal(start)
dend = dt.dt_to_decimal(end)
dstart = dt.dt_to_decimal(rstart)
dend = dt.dt_to_decimal(rend)
codes = {}
@ -52,8 +53,8 @@ def make_report(yesterday=None, start_hour=0, hours=24, percentile=90, store=Fal
report = False
req = req_dict['request_id']
raws = models.RawData.objects.filter(request_id=req)\
.exclude(event='compute.instance.exists')\
.order_by('when')
.exclude(event='compute.instance.exists')\
.order_by('when')
start = None
err = None
@ -74,7 +75,7 @@ def make_report(yesterday=None, start_hour=0, hours=24, percentile=90, store=Fal
break
if raw.image_type:
image_type_num |= raw.image_type
image_type_num |= raw.image_type
image = "?"
if image_type.isset(image_type_num, image_type.BASE_IMAGE):
@ -106,8 +107,8 @@ def make_report(yesterday=None, start_hour=0, hours=24, percentile=90, store=Fal
# Summarize the results ...
report = []
pct = (float(100 - percentile) / 2.0) / 100.0
details = {'percentile': percentile, 'pct': pct, 'hours': hours,
'start': start, 'end': end}
details = {'percentile': percentile, 'pct': pct, 'hours': hours,
'start': float(dstart), 'end': float(dend)}
report.append(details)
cols = ["Operation", "Image", "Min*", "Max*", "Avg*",
@ -143,13 +144,13 @@ def make_report(yesterday=None, start_hour=0, hours=24, percentile=90, store=Fal
_fmax = dt.sec_to_str(_max)
_favg = dt.sec_to_str(_avg)
report.add_row([operation, image, _fmin, _fmax, _favg, count,
failure_count, failure_percentage])
report.append([operation, image, _fmin, _fmax, _favg, count,
failure_count, failure_percentage])
details['total'] = total
details['failures'] = failures
details['failure_total'] = failure_total
details['failure_rate'] = (float(failure_total)/float(total)) * 100.0
return report
return (rstart, rend, report)
def valid_date(date):
@ -157,39 +158,57 @@ def valid_date(date):
t = time.strptime(date, "%Y-%m-%d")
return datetime.datetime(*t[:6])
except Exception, e:
raise argparse.ArgumentTypeError("'%s' is not in YYYY-MM-DD format." % date)
raise argparse.ArgumentTypeError(
"'%s' is not in YYYY-MM-DD format." % date)
if __name__ == '__main__':
parser = argparse.ArgumentParser('StackTach Nova Usage Summary Report')
parser.add_argument('--utcdate', help='Report start date YYYY-MM-DD. Default yesterday midnight.',
type=valid_date, default=None)
parser.add_argument('--hours', help='Report span in hours. Default: 24', default=24, type=int)
parser.add_argument('--start_hour', help='Starting hour 0-23. Default: 0', default=0, type=int)
parser.add_argument('--percentile', help='Percentile for timings. Default: 90', default=90, type=int)
parser.add_argument('--store', help='Store report in database. Default: False', default=False,
action="store_true")
parser.add_argument('--silent', help="Do not show summary report. Default: False", default=False,
action="store_true")
parser.add_argument('--utcdate',
help='Report start date YYYY-MM-DD. Default yesterday midnight.',
type=valid_date, default=None)
parser.add_argument('--hours',
help='Report span in hours. Default: 24', default=24,
type=int)
parser.add_argument('--start_hour',
help='Starting hour 0-23. Default: 0', default=0,
type=int)
parser.add_argument('--percentile',
help='Percentile for timings. Default: 90', default=90,
type=int)
parser.add_argument('--store',
help='Store report in database. Default: False',
default=False, action="store_true")
parser.add_argument('--silent',
help="Do not show summary report. Default: False",
default=False, action="store_true")
args = parser.parse_args()
yesterday = args.utcdate
percentile = args.percentile
hours = args.hours
start_hour = args.start_hour
store_report = args.store
print args
sys.exit(1)
raw_report = make_report(yesterday, start_hour, hours, percentile, args['store'])
start, end, raw_report = make_report(yesterday, start_hour, hours,
percentile, store_report)
details = raw_report[0]
pct = details['pct']
if not args.show:
if store_report:
values = {'json': json.dumps(raw_report),
'created': float(dt.dt_to_decimal(datetime.datetime.utcnow())),
'period_start': start,
'period_end': end,
'version': 1,
'name': 'summary report'}
report = models.JsonReport(**values)
report.save()
print "Report stored (id=%d)" % report.id
if args.silent:
sys.exit(1)
details = raw_report[0]
percentile = details['percentile']
pct = details['pct']
start = details['start']
end = details['end']
print "Report for %s to %s" % (start, end)
cols = raw_report[1]
@ -203,11 +222,13 @@ if __name__ == '__main__':
print "* Using %d-th percentile for results (+/-%.1f%% cut)" % \
(percentile, pct * 100.0)
for row in raw_report[2:]:
p.add_row(row)
frow = row[:]
frow[-1] = "%.1f%%" % (row[-1] * 100.0)
p.add_row(frow)
print p
total = details['total']
failure_total = details['failure_total']
failure_rate = details['failure_rate']
print "Total: %d, Failures: %d, Failure Rate: %.1f%%" % \
(total, failure_total,
(float(failure_total)/float(total)) * 100.0)
(total, failure_total, failure_rate)

2
settings.py
View File

@ -19,7 +19,7 @@ except ImportError:
db_password = os.environ['STACKTACH_DB_PASSWORD']
install_dir = os.environ['STACKTACH_INSTALL_DIR']
DEBUG = True
DEBUG = False
TEMPLATE_DEBUG = DEBUG
ADMINS = (

2
stacktach/models.py
View File

@ -161,7 +161,7 @@ class JsonReport(models.Model):
via stacky/rest. All DateTimes are UTC."""
period_start = models.DateTimeField(db_index=True)
period_end = models.DateTimeField(db_index=True)
created = models.DateTimeField(db_index=True)
created = models.DecimalField(max_digits=20, decimal_places=6, db_index=True)
name = models.CharField(max_length=50, db_index=True)
version = models.IntegerField(default=1)
json = models.TextField()

28
stacktach/stacky_server.py
View File

@ -4,6 +4,7 @@ import json
from django.db.models import Q
from django.http import HttpResponse
from django.shortcuts import get_object_or_404
import datetime_to_decimal as dt
import models
@ -375,3 +376,30 @@ def do_list_usage_exists(request):
exist.status])
return rsp(results)
def do_jsonreports(request):
yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1)
now = datetime.datetime.utcnow()
yesterday = dt.dt_to_decimal(yesterday)
now = dt.dt_to_decimal(now)
_from = float(request.GET.get('created_from', yesterday))
_to = float(request.GET.get('created_to', now))
reports = models.JsonReport.objects.filter(created__gte=_from,
created__lte=_to)
results = []
results.append(['Id', 'Start', 'End', 'Created', 'Name', 'Version'])
for report in reports:
results.append([report.id,
float(dt.dt_to_decimal(report.period_start)),
float(dt.dt_to_decimal(report.period_end)),
float(report.created),
report.name,
report.version])
return rsp(results)
def do_jsonreport(request, report_id):
report_id = int(report_id)
report = get_object_or_404(models.JsonReport, pk=report_id)
return rsp(report.json)

3
stacktach/urls.py
View File

@ -12,6 +12,9 @@ urlpatterns = patterns('',
url(r'stacky/timings/uuid/$', 'stacktach.stacky_server.do_timings_uuid'),
url(r'stacky/summary/$', 'stacktach.stacky_server.do_summary'),
url(r'stacky/request/$', 'stacktach.stacky_server.do_request'),
url(r'stacky/reports/$', 'stacktach.stacky_server.do_jsonreports'),
url(r'stacky/report/(?P<report_id>\d+)/$',
'stacktach.stacky_server.do_jsonreport'),
url(r'stacky/show/(?P<event_id>\d+)/$',
'stacktach.stacky_server.do_show'),
url(r'stacky/watch/(?P<deployment_id>\d+)/$',