x/stacktach
Code Issues Proposed changes

incorporated the review changes and optimized the old_images_query

Browse Source
This commit is contained in:
Manali Latkar
2013-09-19 16:43:53 +05:30
parent 59ee803c81
commit b76e245c12
1 changed files with 42 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
reports/glance_usage_audit.py
View File

@@ -10,33 +10,32 @@ from reports import usage_audit
from stacktach import models from stacktach import models
from stacktach import datetime_to_decimal as dt from stacktach import datetime_to_decimal as dt
OLD_IMAGES_QUERY = """
OLD_LAUNCHES_QUERY = """ select * from stacktach_imageusage left join stacktach_imagedeletes
select * from stacktach_imageusage where on (stacktach_imageusage.uuid = stacktach_imagedeletes.uuid and
created_at is not null and deleted_at < %s)
created_at < %s and where stacktach_imagedeletes.id IS NULL
uuid not in and created_at is not null and created_at < %s;"""
(select distinct(uuid)
from stacktach_imagedeletes where
deleted_at < %s);"""
def audit_usages_to_exists(exists, usages): def audit_usages_to_exists(exists, usages):
# checks if all exists correspond to the given usages # checks if all exists correspond to the given usages
fails = [] fails = []
for (uuid, launches) in usages.items(): for (uuid, images) in usages.items():
if uuid not in exists: if uuid not in exists:
msg = "No exists for usage (%s)" % uuid msg = "No exists for usage (%s)" % uuid
fails.append(['Usage', launches[0]['id'], msg]) fails.append(['Usage', images[0]['id'], msg])
return fails return fails
def _get_new_launches(beginning, ending):
def _get_new_images(beginning, ending):
filters = { filters = {
'created_at__gte': beginning, 'created_at__gte': beginning,
'created_at__lte': ending, 'created_at__lte': ending,
} }
return models.ImageUsage.objects.filter(**filters) return models.ImageUsage.objects.filter(**filters)
def _get_exists(beginning, ending): def _get_exists(beginning, ending):
filters = { filters = {
'audit_period_beginning': beginning, 'audit_period_beginning': beginning,
@@ -45,6 +44,7 @@ def _get_exists(beginning, ending):
} }
return models.ImageExists.objects.filter(**filters) return models.ImageExists.objects.filter(**filters)
def valid_datetime(d): def valid_datetime(d):
try: try:
t = datetime.datetime.strptime(d, "%Y-%m-%d %H:%M:%S") t = datetime.datetime.strptime(d, "%Y-%m-%d %H:%M:%S")
@@ -62,25 +62,26 @@ def audit_for_period(beginning, ending):
verify_detail) = _verifier_audit_for_day(beginning_decimal, verify_detail) = _verifier_audit_for_day(beginning_decimal,
ending_decimal, ending_decimal,
models.ImageExists) models.ImageExists)
detail, new_count, old_count = _launch_audit_for_period(beginning_decimal, detail, new_count, old_count = _image_audit_for_period(beginning_decimal,
ending_decimal) ending_decimal)
summary = { summary = {
'verifier': verify_summary, 'verifier': verify_summary,
'launch_summary': { 'image_summary': {
'new_launches': new_count, 'new_images': new_count,
'old_launches': old_count, 'old_images': old_count,
'failures': len(detail) 'failures': len(detail)
}, },
} }
details = { details = {
'exist_fails': verify_detail, 'exist_fails': verify_detail,
'launch_fails': detail, 'image_fails': detail,
} }
return summary, details return summary, details
def _verifier_audit_for_day(beginning, ending, exists_model): def _verifier_audit_for_day(beginning, ending, exists_model):
summary = {} summary = {}
period = 60*60*24-0.000001 period = 60*60*24-0.000001
@@ -92,8 +93,8 @@ def _verifier_audit_for_day(beginning, ending, exists_model):
'audit_period_ending': F('audit_period_beginning') + period 'audit_period_ending': F('audit_period_beginning') + period
} }
instant_exists = exists_model.objects.filter(**filters) exists = exists_model.objects.filter(**filters)
summary['exists'] = _audit_for_exists(instant_exists) summary['exists'] = _audit_for_exists(exists)
filters = { filters = {
'raw__when__gte': beginning, 'raw__when__gte': beginning,
@@ -106,6 +107,7 @@ def _verifier_audit_for_day(beginning, ending, exists_model):
detail.append(['Exist', exist.id, exist.fail_reason]) detail.append(['Exist', exist.id, exist.fail_reason])
return summary, detail return summary, detail
def _audit_for_exists(exists_query): def _audit_for_exists(exists_query):
(verified, reconciled, (verified, reconciled,
fail, pending, verifying) = usage_audit._status_queries(exists_query) fail, pending, verifying) = usage_audit._status_queries(exists_query)
@@ -129,29 +131,28 @@ def _audit_for_exists(exists_query):
} }
return report return report
def _launch_audit_for_period(beginning, ending):
launches_dict = {}
new_launches = _get_new_launches(beginning, ending)
for launch in new_launches:
uuid = launch.uuid
l = {'id': launch.id, 'created_at': launch.created_at}
if uuid in launches_dict:
launches_dict[uuid].append(l)
else:
launches_dict[uuid] = [l, ]
# NOTE (apmelton) def _image_audit_for_period(beginning, ending):
images_dict = {}
new_images = _get_new_images(beginning, ending)
for image in new_images:
uuid = image.uuid
l = {'id': image.id, 'created_at': image.created_at}
if uuid in images_dict:
images_dict[uuid].append(l)
else:
images_dict[uuid] = [l, ]
# Django's safe substitution doesn't allow dict substitution... # Django's safe substitution doesn't allow dict substitution...
# Thus, we send it 'beginning' three times... # Thus, we send it 'beginning' two times...
old_launches = models.ImageUsage.objects\ old_images = models.ImageUsage.objects\
.raw(OLD_LAUNCHES_QUERY, .raw(OLD_IMAGES_QUERY,
[beginning, beginning]) [beginning, beginning])
old_launches_dict = {} old_images_dict = {}
for launch in old_launches: for image in old_images:
uuid = launch.uuid uuid = image.uuid
l = {'id': launch.id, 'created_at': launch.created_at} l = {'id': image.id, 'created_at': image.created_at}
old_launches_dict[uuid] = l old_images_dict[uuid] = l
exists_dict = {} exists_dict = {}
exists = _get_exists(beginning, ending) exists = _get_exists(beginning, ending)
@@ -165,8 +166,8 @@ def _launch_audit_for_period(beginning, ending):
else: else:
exists_dict[uuid] = [e, ] exists_dict[uuid] = [e, ]
launch_to_exists_fails = audit_usages_to_exists(exists_dict,launches_dict) image_to_exists_fails = audit_usages_to_exists(exists_dict,images_dict)
return launch_to_exists_fails, new_launches.count(), len(old_launches_dict) return image_to_exists_fails, new_images.count(), len(old_images_dict)
def store_results(start, end, summary, details): def store_results(start, end, summary, details):
@@ -187,7 +188,7 @@ def make_json_report(summary, details):
report = [{'summary': summary}, report = [{'summary': summary},
['Object', 'ID', 'Error Description']] ['Object', 'ID', 'Error Description']]
report.extend(details['exist_fails']) report.extend(details['exist_fails'])
report.extend(details['launch_fails']) report.extend(details['image_fails'])
return json.dumps(report) return json.dumps(report)