From e426aa5e84d939249bbbf6d9cb5dbe09783710e7 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Fri, 22 Feb 2013 13:41:00 -0500 Subject: [PATCH 1/5] Adding dbapi query limit and offset --- stacktach/dbapi.py | 59 +++++++++++-------- stacktach/models.py | 20 +++---- tests/unit/test_dbapi.py | 117 ++++++++++++++++++++++++++++++++++++++ tests/unit/test_worker.py | 1 + 4 files changed, 163 insertions(+), 34 deletions(-) diff --git a/stacktach/dbapi.py b/stacktach/dbapi.py index 98003a8..1b2cbb1 100644 --- a/stacktach/dbapi.py +++ b/stacktach/dbapi.py @@ -52,14 +52,8 @@ def api_call(func): @api_call def list_usage_launches(request): - filter_args = _get_filter_args(models.InstanceUsage, request) - - if len(filter_args) > 0: - objects = models.InstanceUsage.objects.filter(**filter_args) - else: - objects = models.InstanceUsage.objects.all() - - dicts = _convert_model_list(objects.order_by("launched_at")) + objects = get_db_objects(models.InstanceUsage, request, 'launched_at') + dicts = _convert_model_list(objects) return {'launches': dicts} @@ -70,14 +64,8 @@ def get_usage_launch(request, launch_id): @api_call def list_usage_deletes(request): - filter_args = _get_filter_args(models.InstanceDeletes, request) - - if len(filter_args) > 0: - objects = models.InstanceDeletes.objects.filter(**filter_args) - else: - objects = models.InstanceDeletes.objects.all() - - dicts = _convert_model_list(objects.order_by("launched_at")) + objects = get_db_objects(models.InstanceDeletes, request, 'launched_at') + dicts = _convert_model_list(objects) return {'deletes': dicts} @@ -88,14 +76,8 @@ def get_usage_delete(request, delete_id): @api_call def list_usage_exists(request): - filter_args = _get_filter_args(models.InstanceExists, request) - - if len(filter_args) > 0: - objects = models.InstanceExists.objects.filter(**filter_args) - else: - objects = models.InstanceExists.objects.all() - - dicts = _convert_model_list(objects.order_by("id")) + objects = get_db_objects(models.InstanceExists, request, 'id') + dicts = _convert_model_list(objects) return {'exists': dicts} @@ -145,6 +127,35 @@ def _get_filter_args(klass, request): return filter_args +def get_db_objects(klass, request, default_order_by, direction='asc'): + filter_args = _get_filter_args(klass, request) + + if len(filter_args) > 0: + objects = klass.objects.filter(**filter_args) + else: + objects = klass.objects.all() + + order_by = request.GET.get('order_by', default_order_by) + _check_has_field(klass, order_by) + + direction = request.GET.get('direction', direction) + if direction == 'desc': + order_by = '-%s' % order_by + + offset = request.GET.get('offset') + limit = request.GET.get('limit') + if offset: + start = int(offset) + else: + start = None + offset = 0 + if limit: + end = int(offset) + int(limit) + else: + end = None + return objects.order_by(order_by)[start:end] + + def _convert_model(model): model_dict = model_to_dict(model) for key in model_dict: diff --git a/stacktach/models.py b/stacktach/models.py index 9a09059..222a933 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -87,11 +87,11 @@ class InstanceUsage(models.Model): class InstanceDeletes(models.Model): instance = models.CharField(max_length=50, null=True, - blank=True, db_index=True) + blank=True, db_index=True) launched_at = models.DecimalField(null=True, max_digits=20, - decimal_places=6) + decimal_places=6) deleted_at = models.DecimalField(null=True, max_digits=20, - decimal_places=6) + decimal_places=6) raw = models.ForeignKey(RawData, null=True) @@ -109,13 +109,13 @@ class InstanceExists(models.Model): launched_at = models.DecimalField(null=True, max_digits=20, decimal_places=6) deleted_at = models.DecimalField(null=True, max_digits=20, - decimal_places=6) - message_id = models.CharField(max_length=50, null=True, - blank=True, db_index=True) - instance_type_id = models.CharField(max_length=50, - null=True, - blank=True, - db_index=True) + decimal_places=6) + message_id = models.CharField(max_length=50, null=True, + blank=True, db_index=True) + instance_type_id = models.CharField(max_length=50, + null=True, + blank=True, + db_index=True) status = models.CharField(max_length=50, db_index=True, choices=STATUS_CHOICES, default=PENDING) diff --git a/tests/unit/test_dbapi.py b/tests/unit/test_dbapi.py index 8d18e08..3f993d3 100644 --- a/tests/unit/test_dbapi.py +++ b/tests/unit/test_dbapi.py @@ -20,6 +20,8 @@ class StacktachRawParsingTestCase(unittest.TestCase): fake_model = self.mox.CreateMockAnything() fake_meta = self.mox.CreateMockAnything() fake_model._meta = fake_meta + fake_orm = self.mox.CreateMockAnything() + fake_model.objects = fake_orm return fake_model def test_get_filter_args(self): @@ -82,3 +84,118 @@ class StacktachRawParsingTestCase(unittest.TestCase): fake_model, fake_request) self.mox.VerifyAll() + + def test_get_db_objects(self): + fake_model = self.make_fake_model() + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {} + self.mox.StubOutWithMock(dbapi, '_get_filter_args') + dbapi._get_filter_args(fake_model, fake_request).AndReturn({}) + self.mox.StubOutWithMock(dbapi, '_check_has_field') + dbapi._check_has_field(fake_model, 'id') + result = self.mox.CreateMockAnything() + fake_model.objects.all().AndReturn(result) + result.order_by('id').AndReturn(result) + result.__getitem__(slice(None, None, None)).AndReturn(result) + self.mox.ReplayAll() + + query_result = dbapi.get_db_objects(fake_model, fake_request, 'id') + self.assertEquals(query_result, result) + + self.mox.VerifyAll() + + def test_get_db_objects_desc(self): + fake_model = self.make_fake_model() + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'direction': 'desc'} + self.mox.StubOutWithMock(dbapi, '_get_filter_args') + dbapi._get_filter_args(fake_model, fake_request).AndReturn({}) + self.mox.StubOutWithMock(dbapi, '_check_has_field') + dbapi._check_has_field(fake_model, 'id') + result = self.mox.CreateMockAnything() + fake_model.objects.all().AndReturn(result) + result.order_by('-id').AndReturn(result) + result.__getitem__(slice(None, None, None)).AndReturn(result) + self.mox.ReplayAll() + + query_result = dbapi.get_db_objects(fake_model, fake_request, 'id') + self.assertEquals(query_result, result) + + self.mox.VerifyAll() + + def test_get_db_objects_limit(self): + fake_model = self.make_fake_model() + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'limit': 1} + self.mox.StubOutWithMock(dbapi, '_get_filter_args') + dbapi._get_filter_args(fake_model, fake_request).AndReturn({}) + self.mox.StubOutWithMock(dbapi, '_check_has_field') + dbapi._check_has_field(fake_model, 'id') + result = self.mox.CreateMockAnything() + fake_model.objects.all().AndReturn(result) + result.order_by('id').AndReturn(result) + result.__getitem__(slice(None, 1, None)).AndReturn(result) + self.mox.ReplayAll() + + query_result = dbapi.get_db_objects(fake_model, fake_request, 'id') + self.assertEquals(query_result, result) + + self.mox.VerifyAll() + + def test_get_db_objects_offset(self): + fake_model = self.make_fake_model() + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'offset': 1} + self.mox.StubOutWithMock(dbapi, '_get_filter_args') + dbapi._get_filter_args(fake_model, fake_request).AndReturn({}) + self.mox.StubOutWithMock(dbapi, '_check_has_field') + dbapi._check_has_field(fake_model, 'id') + result = self.mox.CreateMockAnything() + fake_model.objects.all().AndReturn(result) + result.order_by('id').AndReturn(result) + result.__getitem__(slice(1, None, None)).AndReturn(result) + self.mox.ReplayAll() + + query_result = dbapi.get_db_objects(fake_model, fake_request, 'id') + self.assertEquals(query_result, result) + + self.mox.VerifyAll() + + def test_get_db_objects_offset_and_limit(self): + fake_model = self.make_fake_model() + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'offset': 2, 'limit': 2} + self.mox.StubOutWithMock(dbapi, '_get_filter_args') + dbapi._get_filter_args(fake_model, fake_request).AndReturn({}) + self.mox.StubOutWithMock(dbapi, '_check_has_field') + dbapi._check_has_field(fake_model, 'id') + result = self.mox.CreateMockAnything() + fake_model.objects.all().AndReturn(result) + result.order_by('id').AndReturn(result) + result.__getslice__(2, 4).AndReturn(result) + self.mox.ReplayAll() + + query_result = dbapi.get_db_objects(fake_model, fake_request, 'id') + self.assertEquals(query_result, result) + + self.mox.VerifyAll() + + def test_get_db_objects_with_filter(self): + fake_model = self.make_fake_model() + fake_request = self.mox.CreateMockAnything() + filters = {'instance': INSTANCE_ID_1} + fake_request.GET = filters + self.mox.StubOutWithMock(dbapi, '_get_filter_args') + dbapi._get_filter_args(fake_model, fake_request).AndReturn(filters) + self.mox.StubOutWithMock(dbapi, '_check_has_field') + dbapi._check_has_field(fake_model, 'id') + result = self.mox.CreateMockAnything() + fake_model.objects.filter(**filters).AndReturn(result) + result.order_by('id').AndReturn(result) + result.__getitem__(slice(None, None, None)).AndReturn(result) + self.mox.ReplayAll() + + query_result = dbapi.get_db_objects(fake_model, fake_request, 'id') + self.assertEquals(query_result, result) + + self.mox.VerifyAll() diff --git a/tests/unit/test_worker.py b/tests/unit/test_worker.py index ada86bb..3d2d3f5 100644 --- a/tests/unit/test_worker.py +++ b/tests/unit/test_worker.py @@ -9,6 +9,7 @@ import mox from stacktach import db, views import worker.worker as worker + class NovaConsumerTestCase(unittest.TestCase): def setUp(self): self.mox = mox.Mox() From 5df346da894c746dac9db4e5e2a70c97f5f73eeb Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Tue, 26 Feb 2013 15:35:52 -0500 Subject: [PATCH 2/5] Testing and refactoring stacky_server --- stacktach/stacky_server.py | 88 ++-- tests/integration/settings.py | 2 +- tests/unit/test_stacky_server.py | 730 +++++++++++++++++++++++++++++++ 3 files changed, 766 insertions(+), 54 deletions(-) create mode 100644 tests/unit/test_stacky_server.py diff --git a/stacktach/stacky_server.py b/stacktach/stacky_server.py index 5543aa3..f5d5de2 100644 --- a/stacktach/stacky_server.py +++ b/stacktach/stacky_server.py @@ -39,8 +39,7 @@ def get_deployments(): def get_timings_for_uuid(uuid): lifecycles = models.Lifecycle.objects.filter(instance=uuid) - results = [] - results.append(["?", "Event", "Time (secs)"]) + results = [["?", "Event", "Time (secs)"]] for lc in lifecycles: timings = models.Timing.objects.filter(lifecycle=lc) if not timings: @@ -72,14 +71,14 @@ def sec_to_time(diff): return "%dd %02d:%02d:%02d%s" % (days, hours, minutes, seconds, usec) -def rsp(data): - return HttpResponse(json.dumps(data), content_type="application/json") +def rsp(data, status=200): + return HttpResponse(json.dumps(data), content_type="application/json", + status=status) def do_deployments(request): deployments = get_deployments() - results = [] - results.append(["#", "Name"]) + results = [["#", "Name"]] for deployment in deployments: results.append([deployment.id, deployment.name]) return rsp(results) @@ -87,8 +86,7 @@ def do_deployments(request): def do_events(request): events = get_event_names() - results = [] - results.append(["Event Name"]) + results = [["Event Name"]] for event in events: results.append([event['event']]) return rsp(results) @@ -96,8 +94,7 @@ def do_events(request): def do_hosts(request): hosts = get_host_names() - results = [] - results.append(["Host Name"]) + results = [["Host Name"]] for host in hosts: results.append([host['host']]) return rsp(results) @@ -105,17 +102,15 @@ def do_hosts(request): def do_uuid(request): uuid = str(request.GET['uuid']) - related = models.RawData.objects.select_related(). \ - filter(instance=uuid).order_by('when') - results = [] - results.append(["#", "?", "When", "Deployment", "Event", "Host", - "State", "State'", "Task'"]) + related = models.RawData.objects.select_related().filter(instance=uuid)\ + .order_by('when') + results = [["#", "?", "When", "Deployment", "Event", "Host", "State", + "State'", "Task'"]] for e in related: when = dt.dt_from_decimal(e.when) results.append([e.id, routing_key_type(e.routing_key), str(when), - e.deployment.name, e.event, - e.host, - e.state, e.old_state, e.old_task]) + e.deployment.name, e.event, e.host, e.state, + e.old_state, e.old_task]) return rsp(results) @@ -126,11 +121,10 @@ def do_timings_uuid(request): def do_timings(request): name = request.GET['name'] - results = [] - results.append([name, "Time"]) - timings = models.Timing.objects.select_related().filter(name=name) \ - .exclude(Q(start_raw=None) | Q(end_raw=None)) \ - .order_by('diff') + results = [[name, "Time"]] + timings = models.Timing.objects.select_related().filter(name=name)\ + .exclude(Q(start_raw=None) | Q(end_raw=None))\ + .order_by('diff') for t in timings: results.append([t.lifecycle.instance, sec_to_time(t.diff)]) @@ -145,8 +139,7 @@ def do_summary(request): if ev.endswith('.start'): interesting.append(ev[:-len('.start')]) - results = [] - results.append(["Event", "N", "Min", "Max", "Avg"]) + results = [["Event", "N", "Min", "Max", "Avg"]] for name in interesting: timings = models.Timing.objects.filter(name=name) \ @@ -168,7 +161,7 @@ def do_summary(request): _max = max(_max, seconds) results.append([name, int(num), sec_to_time(_min), - sec_to_time(_max), sec_to_time(int(total/num)) ]) + sec_to_time(_max), sec_to_time(int(total / num))]) return rsp(results) @@ -176,15 +169,13 @@ def do_request(request): request_id = request.GET['request_id'] events = models.RawData.objects.filter(request_id=request_id) \ .order_by('when') - results = [] - results.append(["#", "?", "When", "Deployment", "Event", "Host", - "State", "State'", "Task'"]) + results = [["#", "?", "When", "Deployment", "Event", "Host", + "State", "State'", "Task'"]] for e in events: when = dt.dt_from_decimal(e.when) results.append([e.id, routing_key_type(e.routing_key), str(when), - e.deployment.name, e.event, - e.host, e.state, - e.old_state, e.old_task]) + e.deployment.name, e.event, e.host, e.state, + e.old_state, e.old_task]) return rsp(results) @@ -229,8 +220,6 @@ def do_watch(request, deployment_id): deployment_map[d.id] = d events = get_event_names() max_event_width = max([len(event['event']) for event in events]) - hosts = get_host_names() - max_host_width = max([len(host['host']) for host in hosts]) base_events = models.RawData.objects.order_by('when') if deployment_id > 0: @@ -260,8 +249,6 @@ def do_watch(request, deployment_id): events = base_events.filter(when__lte=dec_now) c = [10, 1, 15, 20, max_event_width, 36] - header = ("+%s" * len(c)) + "+" - splat = header.replace("+", "|") results = [] @@ -283,20 +270,19 @@ def do_watch(request, deployment_id): def do_kpi(request, tenant_id=None): yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1) yesterday = dt.dt_to_decimal(yesterday) - trackers = models.RequestTracker.objects.select_related() \ - .exclude(last_timing=None) \ - .exclude(start__lt=yesterday) \ - .order_by('duration') + trackers = models.RequestTracker.objects.select_related()\ + .exclude(last_timing=None)\ + .exclude(start__lt=yesterday)\ + .order_by('duration') - results = [] - results.append(["Event", "Time", "UUID", "Deployment"]) + results = [["Event", "Time", "UUID", "Deployment"]] for track in trackers: end_event = track.last_timing.end_raw event = end_event.event[:-len(".end")] uuid = track.lifecycle.instance - if tenant_id == None or (tenant_id == end_event.tenant): + if tenant_id is None or (tenant_id == end_event.tenant): results.append([event, sec_to_time(track.duration), - uuid, end_event.deployment.name]) + uuid, end_event.deployment.name]) return rsp(results) @@ -311,8 +297,7 @@ def do_list_usage_launches(request): else: launches = models.InstanceUsage.objects.all() - results = [] - results.append(["UUID", "Launched At", "Instance Type Id"]) + results = [["UUID", "Launched At", "Instance Type Id"]] for launch in launches: launched = None @@ -334,8 +319,7 @@ def do_list_usage_deletes(request): else: deletes = models.InstanceDeletes.objects.all() - results = [] - results.append(["UUID", "Launched At", "Deleted At"]) + results = [["UUID", "Launched At", "Deleted At"]] for delete in deletes: launched = None @@ -360,9 +344,8 @@ def do_list_usage_exists(request): else: exists = models.InstanceExists.objects.all() - results = [] - results.append(["UUID", "Launched At", "Deleted At", "Instance Type Id", - "Message ID", "Status"]) + results = [["UUID", "Launched At", "Deleted At", "Instance Type Id", + "Message ID", "Status"]] for exist in exists: launched = None @@ -387,8 +370,7 @@ def do_jsonreports(request): _to = request.GET.get('created_to', now) reports = models.JsonReport.objects.filter(created__gte=_from, created__lte=_to) - results = [] - results.append(['Id', 'Start', 'End', 'Created', 'Name', 'Version']) + results = [['Id', 'Start', 'End', 'Created', 'Name', 'Version']] for report in reports: results.append([report.id, float(dt.dt_to_decimal(report.period_start)), diff --git a/tests/integration/settings.py b/tests/integration/settings.py index fff7c2e..8889b19 120000 --- a/tests/integration/settings.py +++ b/tests/integration/settings.py @@ -1 +1 @@ -settings.py \ No newline at end of file +../../settings.py \ No newline at end of file diff --git a/tests/unit/test_stacky_server.py b/tests/unit/test_stacky_server.py new file mode 100644 index 0000000..4f78414 --- /dev/null +++ b/tests/unit/test_stacky_server.py @@ -0,0 +1,730 @@ +import datetime +import json +import unittest + +import mox + +from stacktach import datetime_to_decimal as dt +from stacktach import models +from stacktach import stacky_server +import utils +from utils import INSTANCE_ID_1 +from utils import INSTANCE_ID_2 + + +class StackyServerTestCase(unittest.TestCase): + def setUp(self): + self.mox = mox.Mox() + self.mox.StubOutWithMock(models, 'RawData', use_mock_anything=True) + models.RawData.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'Deployment', use_mock_anything=True) + models.Deployment.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'Lifecycle', use_mock_anything=True) + models.Lifecycle.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'Timing', use_mock_anything=True) + models.Timing.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'RequestTracker', + use_mock_anything=True) + models.RequestTracker.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'InstanceUsage', + use_mock_anything=True) + models.InstanceUsage.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'InstanceDeletes', + use_mock_anything=True) + models.InstanceDeletes.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'InstanceExists', + use_mock_anything=True) + models.InstanceExists.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'JsonReport', use_mock_anything=True) + models.JsonReport.objects = self.mox.CreateMockAnything() + + def tearDown(self): + self.mox.UnsetStubs() + + def _create_raw(self): + raw = self.mox.CreateMockAnything() + raw.when = utils.decimal_utc() + raw.instance = INSTANCE_ID_1 + raw.id = 1 + raw.routing_key = 'monitor.info' + raw.deployment = self.mox.CreateMockAnything() + raw.deployment.id = 1 + raw.deployment.name = 'deployment' + raw.event = 'test.start' + raw.host = 'example.com' + raw.state = 'active' + raw.old_state = None + raw.old_task = None + raw.publisher = "api.example.com" + raw.service = 'api' + raw.host = 'example.com' + raw.request_id = 'req-1' + raw.json = '{"key": "value"}' + return raw + + def test_get_event_names(self): + result = self.mox.CreateMockAnything() + models.RawData.objects.values('event').AndReturn(result) + result.distinct().AndReturn(result) + self.mox.ReplayAll() + + event_names = stacky_server.get_event_names() + self.assertEqual(event_names, result) + + self.mox.VerifyAll() + + def test_get_host_names(self): + result = self.mox.CreateMockAnything() + models.RawData.objects.values('host').AndReturn(result) + result.distinct().AndReturn(result) + self.mox.ReplayAll() + + event_names = stacky_server.get_host_names() + self.assertEqual(event_names, result) + + self.mox.VerifyAll() + + def test_get_deployments(self): + result = self.mox.CreateMockAnything() + models.Deployment.objects.all().AndReturn(result) + result.order_by('name').AndReturn(result) + self.mox.ReplayAll() + + event_names = stacky_server.get_deployments() + self.assertEqual(event_names, result) + + self.mox.VerifyAll() + + def test_get_timings_for_uuid_start_only(self): + lc_result = self.mox.CreateMockAnything() + lifecycle = self.mox.CreateMockAnything() + models.Lifecycle.objects.filter(instance=INSTANCE_ID_1)\ + .AndReturn(lc_result) + lc_result.__iter__().AndReturn([lifecycle].__iter__()) + t_result = self.mox.CreateMockAnything() + timing = self.mox.CreateMockAnything() + models.Timing.objects.filter(lifecycle=lifecycle).AndReturn(t_result) + t_result.__iter__().AndReturn([timing].__iter__()) + timing.name = 'name' + timing.start_raw = self.mox.CreateMockAnything() + timing.end_raw = None + timing.diff = None + self.mox.ReplayAll() + + event_names = stacky_server.get_timings_for_uuid(INSTANCE_ID_1) + + self.assertEqual(len(event_names), 2) + self.assertEqual(event_names[0], ['?', 'Event', 'Time (secs)']) + self.assertEqual(event_names[1], ['S', 'name', 'n/a']) + self.mox.VerifyAll() + + def test_get_timings_for_uuid_end_only(self): + lc_result = self.mox.CreateMockAnything() + lifecycle = self.mox.CreateMockAnything() + models.Lifecycle.objects.filter(instance=INSTANCE_ID_1) \ + .AndReturn(lc_result) + lc_result.__iter__().AndReturn([lifecycle].__iter__()) + t_result = self.mox.CreateMockAnything() + timing = self.mox.CreateMockAnything() + models.Timing.objects.filter(lifecycle=lifecycle).AndReturn(t_result) + t_result.__iter__().AndReturn([timing].__iter__()) + timing.name = 'name' + timing.start_raw = None + timing.end_raw = self.mox.CreateMockAnything() + timing.diff = None + self.mox.ReplayAll() + + event_names = stacky_server.get_timings_for_uuid(INSTANCE_ID_1) + + self.assertEqual(len(event_names), 2) + self.assertEqual(event_names[0], ['?', 'Event', 'Time (secs)']) + self.assertEqual(event_names[1], ['E', 'name', 'n/a']) + self.mox.VerifyAll() + + def test_get_timings_for_uuid(self): + lc_result = self.mox.CreateMockAnything() + lifecycle = self.mox.CreateMockAnything() + models.Lifecycle.objects.filter(instance=INSTANCE_ID_1) \ + .AndReturn(lc_result) + lc_result.__iter__().AndReturn([lifecycle].__iter__()) + t_result = self.mox.CreateMockAnything() + timing = self.mox.CreateMockAnything() + models.Timing.objects.filter(lifecycle=lifecycle).AndReturn(t_result) + t_result.__iter__().AndReturn([timing].__iter__()) + timing.name = 'name' + timing.start_raw = self.mox.CreateMockAnything() + timing.end_raw = self.mox.CreateMockAnything() + timing.diff = 20 + self.mox.ReplayAll() + event_names = stacky_server.get_timings_for_uuid(INSTANCE_ID_1) + + self.assertEqual(len(event_names), 2) + self.assertEqual(event_names[0], ['?', 'Event', 'Time (secs)']) + self.assertEqual(event_names[1], ['.', 'name', '0d 00:00:20']) + + self.mox.VerifyAll() + + def test_do_deployments(self): + fake_request = self.mox.CreateMockAnything() + deployment1 = self.mox.CreateMockAnything() + deployment1.id = 1 + deployment1.name = 'dep1' + deployment2 = self.mox.CreateMockAnything() + deployment2.id = 2 + deployment2.name = 'dep2' + deployments = [deployment1, deployment2] + self.mox.StubOutWithMock(stacky_server, 'get_deployments') + stacky_server.get_deployments().AndReturn(deployments) + self.mox.ReplayAll() + + resp = stacky_server.do_deployments(fake_request) + + self.assertEqual(resp.status_code, 200) + json_resp = json.loads(resp.content) + self.assertEqual(len(json_resp), 3) + self.assertEqual(json_resp[0], ['#', 'Name']) + self.assertEqual(json_resp[1], [1, 'dep1']) + self.assertEqual(json_resp[2], [2, 'dep2']) + self.mox.VerifyAll() + + def test_do_events(self): + fake_request = self.mox.CreateMockAnything() + event1 = {'event': 'some.event.1'} + event2 = {'event': 'some.event.2'} + events = [event1, event2] + self.mox.StubOutWithMock(stacky_server, 'get_event_names') + stacky_server.get_event_names().AndReturn(events) + self.mox.ReplayAll() + + resp = stacky_server.do_events(fake_request) + + self.assertEqual(resp.status_code, 200) + json_resp = json.loads(resp.content) + self.assertEqual(len(json_resp), 3) + self.assertEqual(json_resp[0], ['Event Name']) + self.assertEqual(json_resp[1], ['some.event.1']) + self.assertEqual(json_resp[2], ['some.event.2']) + self.mox.VerifyAll() + + def test_do_hosts(self): + fake_request = self.mox.CreateMockAnything() + host1 = {'host': 'www.demo.com'} + host2 = {'host': 'www.example.com'} + hosts = [host1, host2] + self.mox.StubOutWithMock(stacky_server, 'get_host_names') + stacky_server.get_host_names().AndReturn(hosts) + self.mox.ReplayAll() + + resp = stacky_server.do_hosts(fake_request) + + self.assertEqual(resp.status_code, 200) + json_resp = json.loads(resp.content) + self.assertEqual(len(json_resp), 3) + self.assertEqual(json_resp[0], ['Host Name']) + self.assertEqual(json_resp[1], ['www.demo.com']) + self.assertEqual(json_resp[2], ['www.example.com']) + self.mox.VerifyAll() + + def test_do_uuid(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'uuid': INSTANCE_ID_1} + result = self.mox.CreateMockAnything() + models.RawData.objects.select_related().AndReturn(result) + result.filter(instance=INSTANCE_ID_1).AndReturn(result) + result.order_by('when').AndReturn(result) + raw = self._create_raw() + result.__iter__().AndReturn([raw].__iter__()) + self.mox.ReplayAll() + + resp = stacky_server.do_uuid(fake_request) + + self.assertEqual(resp.status_code, 200) + json_resp = json.loads(resp.content) + self.assertEqual(len(json_resp), 2) + header = ["#", "?", "When", "Deployment", "Event", "Host", + "State", "State'", "Task'"] + self.assertEqual(json_resp[0], header) + datetime = dt.dt_from_decimal(raw.when) + body = [1, " ", str(datetime), "deployment", "test.start", + "example.com", "active", None, None] + self.assertEqual(json_resp[1], body) + self.mox.VerifyAll() + + def test_do_timings(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'name': 'test.event'} + results = self.mox.CreateMockAnything() + models.Timing.objects.select_related().AndReturn(results) + results.filter(name='test.event').AndReturn(results) + results.exclude(mox.IgnoreArg()).AndReturn(results) + results.order_by('diff').AndReturn(results) + timing1 = self.mox.CreateMockAnything() + timing1.lifecycle = self.mox.CreateMockAnything() + timing1.lifecycle.instance = INSTANCE_ID_1 + timing1.diff = 10 + timing2 = self.mox.CreateMockAnything() + timing2.lifecycle = self.mox.CreateMockAnything() + timing2.lifecycle.instance = INSTANCE_ID_2 + timing2.diff = 20 + results.__iter__().AndReturn([timing1, timing2].__iter__()) + self.mox.ReplayAll() + + resp = stacky_server.do_timings(fake_request) + + self.assertEqual(resp.status_code, 200) + json_resp = json.loads(resp.content) + self.assertEqual(len(json_resp), 3) + header = ["test.event", "Time"] + self.assertEqual(json_resp[0], header) + self.assertEqual(json_resp[1], [INSTANCE_ID_1, '0d 00:00:10']) + self.assertEqual(json_resp[2], [INSTANCE_ID_2, '0d 00:00:20']) + self.mox.VerifyAll() + + def test_do_summary(self): + fake_request = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(stacky_server, 'get_event_names') + events = [{'event': 'test.start'}, {'event': 'test.end'}] + stacky_server.get_event_names().AndReturn(events) + fake_request.GET = {'name': 'test.event'} + results = self.mox.CreateMockAnything() + models.Timing.objects.filter(name='test').AndReturn(results) + results.exclude(mox.IgnoreArg()).AndReturn(results) + results.exclude(diff__lt=0).AndReturn(results) + timing1 = self.mox.CreateMockAnything() + timing1.lifecycle = self.mox.CreateMockAnything() + timing1.lifecycle.instance = INSTANCE_ID_1 + timing1.diff = 10 + timing2 = self.mox.CreateMockAnything() + timing2.lifecycle = self.mox.CreateMockAnything() + timing2.lifecycle.instance = INSTANCE_ID_2 + timing2.diff = 20 + results.__len__().AndReturn(2) + results.__iter__().AndReturn([timing1, timing2].__iter__()) + self.mox.ReplayAll() + + resp = stacky_server.do_summary(fake_request) + self.assertEqual(resp.status_code, 200) + json_resp = json.loads(resp.content) + self.assertEqual(len(json_resp), 2) + self.assertEqual(json_resp[0], ["Event", "N", "Min", "Max", "Avg"]) + self.assertEqual(json_resp[1], [u'test', 2, u'0d 00:00:10.0', + u'0d 00:00:20.0', u'0d 00:00:15']) + + self.mox.VerifyAll() + + def test_do_request(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'request_id': 'req-1'} + raw = self._create_raw() + results = self.mox.CreateMockAnything() + models.RawData.objects.filter(request_id='req-1').AndReturn(results) + results.order_by('when').AndReturn(results) + results.__iter__().AndReturn([raw].__iter__()) + self.mox.ReplayAll() + + resp = stacky_server.do_request(fake_request) + + self.assertEqual(resp.status_code, 200) + json_resp = json.loads(resp.content) + self.assertEqual(len(json_resp), 2) + self.assertEqual(json_resp[0], ["#", "?", "When", "Deployment", + "Event", "Host", "State", "State'", + "Task'"]) + self.assertEqual(json_resp[1][0], 1) + self.assertEqual(json_resp[1][1], u' ') + self.assertEqual(json_resp[1][2], str(dt.dt_from_decimal(raw.when))) + self.assertEqual(json_resp[1][3], u'deployment') + self.assertEqual(json_resp[1][4], u'test.start') + self.assertEqual(json_resp[1][5], u'example.com') + self.assertEqual(json_resp[1][6], u'active') + self.assertEqual(json_resp[1][7], None) + self.assertEqual(json_resp[1][8], None) + self.mox.VerifyAll() + + def _assert_on_show(self, values, raw): + self.assertEqual(len(values), 12) + self.assertEqual(values[0], ["Key", "Value"]) + self.assertEqual(values[1], ["#", raw.id]) + self.assertEqual(values[2], ["When", + str(dt.dt_from_decimal(raw.when))]) + self.assertEqual(values[3], ["Deployment", raw.deployment.name]) + self.assertEqual(values[4], ["Category", raw.routing_key]) + self.assertEqual(values[5], ["Publisher", raw.publisher]) + self.assertEqual(values[6], ["State", raw.state]) + self.assertEqual(values[7], ["Event", raw.event]) + self.assertEqual(values[8], ["Service", raw.service]) + self.assertEqual(values[9], ["Host", raw.host]) + self.assertEqual(values[10], ["UUID", raw.instance]) + self.assertEqual(values[11], ["Req ID", raw.request_id]) + + def test_do_show(self): + fake_request = self.mox.CreateMockAnything() + raw = self._create_raw() + models.RawData.objects.get(id=1).AndReturn(raw) + self.mox.ReplayAll() + + resp = stacky_server.do_show(fake_request, 1) + + self.assertEqual(resp.status_code, 200) + json_resp = json.loads(resp.content) + self.assertEqual(len(json_resp), 3) + self._assert_on_show(json_resp[0], raw) + self.mox.VerifyAll() + + def test_do_watch(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {} + self.mox.StubOutWithMock(stacky_server, 'get_deployments') + deployment1 = self.mox.CreateMockAnything() + deployment1.id = 1 + deployment1.name = 'dep1' + deployments = [deployment1] + stacky_server.get_deployments().AndReturn(deployments) + self.mox.StubOutWithMock(stacky_server, 'get_event_names') + events = [{'event': 'test.start'}, {'event': 'test.end'}] + stacky_server.get_event_names().AndReturn(events) + results = self.mox.CreateMockAnything() + models.RawData.objects.order_by('when').AndReturn(results) + results.filter(when__gt=mox.IgnoreArg()).AndReturn(results) + results.filter(when__lte=mox.IgnoreArg()).AndReturn(results) + results.__iter__().AndReturn([self._create_raw()].__iter__()) + self.mox.ReplayAll() + + resp = stacky_server.do_watch(fake_request, 0) + self.assertEqual(resp.status_code, 200) + json_resp = json.loads(resp.content) + self.assertEqual(len(json_resp), 3) + self.assertEqual(json_resp[0], [10, 1, 15, 20, 10, 36]) + print json_resp + self.assertEqual(json_resp[1][0][0], 1) + self.assertEqual(json_resp[1][0][1], u' ') + time_str = "%s %s" % (json_resp[1][0][2], json_resp[1][0][3]) + datetime.datetime.strptime(time_str, "%Y-%m-%d %H:%M:%S.%f") + self.assertEqual(json_resp[1][0][4], u'dep1') + self.assertEqual(json_resp[1][0][5], u'test.start') + self.assertEqual(json_resp[1][0][6], u'%s' % INSTANCE_ID_1) + self.mox.VerifyAll() + + def test_do_watch_with_deployment(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'deployment': 1} + self.mox.StubOutWithMock(stacky_server, 'get_deployments') + deployment1 = self.mox.CreateMockAnything() + deployment1.id = 1 + deployment1.name = 'dep1' + deployments = [deployment1] + stacky_server.get_deployments().AndReturn(deployments) + self.mox.StubOutWithMock(stacky_server, 'get_event_names') + events = [{'event': 'test.start'}, {'event': 'test.end'}] + stacky_server.get_event_names().AndReturn(events) + results = self.mox.CreateMockAnything() + models.RawData.objects.order_by('when').AndReturn(results) + results.filter(deployment=1).AndReturn(results) + results.filter(when__gt=mox.IgnoreArg()).AndReturn(results) + results.filter(when__lte=mox.IgnoreArg()).AndReturn(results) + results.__iter__().AndReturn([self._create_raw()].__iter__()) + self.mox.ReplayAll() + + resp = stacky_server.do_watch(fake_request, 1) + self.assertEqual(resp.status_code, 200) + json_resp = json.loads(resp.content) + self.assertEqual(len(json_resp), 3) + self.assertEqual(json_resp[0], [10, 1, 15, 20, 10, 36]) + print json_resp + self.assertEqual(json_resp[1][0][0], 1) + self.assertEqual(json_resp[1][0][1], u' ') + time_str = "%s %s" % (json_resp[1][0][2], json_resp[1][0][3]) + datetime.datetime.strptime(time_str, "%Y-%m-%d %H:%M:%S.%f") + self.assertEqual(json_resp[1][0][4], u'dep1') + self.assertEqual(json_resp[1][0][5], u'test.start') + self.assertEqual(json_resp[1][0][6], u'%s' % INSTANCE_ID_1) + self.mox.VerifyAll() + + def test_do_watch_with_event_name(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'event_name': 'test.start'} + self.mox.StubOutWithMock(stacky_server, 'get_deployments') + deployment1 = self.mox.CreateMockAnything() + deployment1.id = 1 + deployment1.name = 'dep1' + deployments = [deployment1] + stacky_server.get_deployments().AndReturn(deployments) + self.mox.StubOutWithMock(stacky_server, 'get_event_names') + events = [{'event': 'test.start'}, {'event': 'test.end'}] + stacky_server.get_event_names().AndReturn(events) + results = self.mox.CreateMockAnything() + models.RawData.objects.order_by('when').AndReturn(results) + results.filter(event='test.start').AndReturn(results) + results.filter(when__gt=mox.IgnoreArg()).AndReturn(results) + results.filter(when__lte=mox.IgnoreArg()).AndReturn(results) + results.__iter__().AndReturn([self._create_raw()].__iter__()) + self.mox.ReplayAll() + + resp = stacky_server.do_watch(fake_request, 0) + self.assertEqual(resp.status_code, 200) + json_resp = json.loads(resp.content) + self.assertEqual(len(json_resp), 3) + self.assertEqual(json_resp[0], [10, 1, 15, 20, 10, 36]) + print json_resp + self.assertEqual(json_resp[1][0][0], 1) + self.assertEqual(json_resp[1][0][1], u' ') + time_str = "%s %s" % (json_resp[1][0][2], json_resp[1][0][3]) + datetime.datetime.strptime(time_str, "%Y-%m-%d %H:%M:%S.%f") + self.assertEqual(json_resp[1][0][4], u'dep1') + self.assertEqual(json_resp[1][0][5], u'test.start') + self.assertEqual(json_resp[1][0][6], u'%s' % INSTANCE_ID_1) + self.mox.VerifyAll() + + def test_do_kpi(self): + fake_request = self.mox.CreateMockAnything() + results = self.mox.CreateMockAnything() + models.RequestTracker.objects.select_related().AndReturn(results) + results.exclude(last_timing=None).AndReturn(results) + results.exclude(start__lt=mox.IgnoreArg()).AndReturn(results) + results.order_by('duration').AndReturn(results) + tracker = self.mox.CreateMockAnything() + tracker.last_timing = self.mox.CreateMockAnything() + tracker.last_timing.end_raw = self.mox.CreateMockAnything() + tracker.last_timing.end_raw.event = 'test.end' + deployment = self.mox.CreateMockAnything() + deployment.name = 'dep1' + tracker.last_timing.end_raw.deployment = deployment + tracker.lifecycle = self.mox.CreateMockAnything() + tracker.lifecycle.instance = INSTANCE_ID_1 + tracker.duration = 10 + results.__iter__().AndReturn([tracker].__iter__()) + self.mox.ReplayAll() + + resp = stacky_server.do_kpi(fake_request) + self.assertEqual(resp.status_code, 200) + body = resp.content + body = json.loads(body) + self.assertEqual(len(body), 2) + self.assertEqual(body[0], ["Event", "Time", "UUID", "Deployment"]) + time = u'%s' % stacky_server.sec_to_time(10) + self.assertEqual(body[1], [u'test', time, INSTANCE_ID_1, u'dep1']) + + self.mox.VerifyAll() + + def test_do_kpi_with_tenant(self): + fake_request = self.mox.CreateMockAnything() + results = self.mox.CreateMockAnything() + models.RequestTracker.objects.select_related().AndReturn(results) + results.exclude(last_timing=None).AndReturn(results) + results.exclude(start__lt=mox.IgnoreArg()).AndReturn(results) + results.order_by('duration').AndReturn(results) + tracker = self.mox.CreateMockAnything() + tracker.last_timing = self.mox.CreateMockAnything() + tracker.last_timing.end_raw = self.mox.CreateMockAnything() + tracker.last_timing.end_raw.event = 'test.end' + tracker.last_timing.end_raw.tenant = '55555' + deployment = self.mox.CreateMockAnything() + deployment.name = 'dep1' + tracker.last_timing.end_raw.deployment = deployment + tracker.lifecycle = self.mox.CreateMockAnything() + tracker.lifecycle.instance = INSTANCE_ID_1 + tracker.duration = 10 + results.__iter__().AndReturn([tracker].__iter__()) + self.mox.ReplayAll() + + resp = stacky_server.do_kpi(fake_request, '55555') + self.assertEqual(resp.status_code, 200) + body = resp.content + body = json.loads(body) + self.assertEqual(len(body), 2) + self.assertEqual(body[0], ["Event", "Time", "UUID", "Deployment"]) + time = u'%s' % stacky_server.sec_to_time(10) + self.assertEqual(body[1], [u'test', time, INSTANCE_ID_1, u'dep1']) + + self.mox.VerifyAll() + + def test_do_kpi_with_tenant_no_match(self): + fake_request = self.mox.CreateMockAnything() + results = self.mox.CreateMockAnything() + models.RequestTracker.objects.select_related().AndReturn(results) + results.exclude(last_timing=None).AndReturn(results) + results.exclude(start__lt=mox.IgnoreArg()).AndReturn(results) + results.order_by('duration').AndReturn(results) + tracker = self.mox.CreateMockAnything() + tracker.last_timing = self.mox.CreateMockAnything() + tracker.last_timing.end_raw = self.mox.CreateMockAnything() + tracker.last_timing.end_raw.event = 'test.end' + tracker.last_timing.end_raw.tenant = '55556' + deployment = self.mox.CreateMockAnything() + deployment.name = 'dep1' + tracker.last_timing.end_raw.deployment = deployment + tracker.lifecycle = self.mox.CreateMockAnything() + tracker.lifecycle.instance = INSTANCE_ID_1 + tracker.duration = 10 + results.__iter__().AndReturn([tracker].__iter__()) + self.mox.ReplayAll() + + resp = stacky_server.do_kpi(fake_request, '55555') + self.assertEqual(resp.status_code, 200) + body = resp.content + body = json.loads(body) + self.assertEqual(len(body), 1) + + self.mox.VerifyAll() + + def test_do_list_usage_launches(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {} + results = self.mox.CreateMockAnything() + models.InstanceUsage.objects.all().AndReturn(results) + usage = self.mox.CreateMockAnything() + usage.instance = INSTANCE_ID_1 + usage.launched_at = utils.decimal_utc() + usage.instance_type_id = 1 + results.__iter__().AndReturn([usage].__iter__()) + self.mox.ReplayAll() + + resp = stacky_server.do_list_usage_launches(fake_request) + self.assertEqual(resp.status_code, 200) + resp_json = json.loads(resp.content) + self.assertEqual(len(resp_json), 2) + self.assertEqual(resp_json[0], ["UUID", "Launched At", + "Instance Type Id"]) + self.assertEqual(resp_json[1][0], INSTANCE_ID_1) + time_str = dt.dt_from_decimal(usage.launched_at) + self.assertEqual(resp_json[1][1], str(time_str)) + self.assertEqual(resp_json[1][2], 1) + + self.mox.VerifyAll() + + def test_do_list_usage_launches_with_instance(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'instance': INSTANCE_ID_1} + results = self.mox.CreateMockAnything() + models.InstanceUsage.objects.filter(instance=INSTANCE_ID_1)\ + .AndReturn(results) + usage = self.mox.CreateMockAnything() + usage.instance = INSTANCE_ID_1 + usage.launched_at = utils.decimal_utc() + usage.instance_type_id = 1 + results.__iter__().AndReturn([usage].__iter__()) + self.mox.ReplayAll() + + resp = stacky_server.do_list_usage_launches(fake_request) + self.assertEqual(resp.status_code, 200) + resp_json = json.loads(resp.content) + self.assertEqual(len(resp_json), 2) + self.assertEqual(resp_json[0], ["UUID", "Launched At", + "Instance Type Id"]) + self.assertEqual(resp_json[1][0], INSTANCE_ID_1) + time_str = dt.dt_from_decimal(usage.launched_at) + self.assertEqual(resp_json[1][1], str(time_str)) + self.assertEqual(resp_json[1][2], 1) + + self.mox.VerifyAll() + + def test_do_list_usage_deletes(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {} + results = self.mox.CreateMockAnything() + models.InstanceDeletes.objects.all().AndReturn(results) + usage = self.mox.CreateMockAnything() + usage.instance = INSTANCE_ID_1 + usage.launched_at = utils.decimal_utc() + usage.deleted_at = usage.launched_at + 10 + results.__iter__().AndReturn([usage].__iter__()) + self.mox.ReplayAll() + + resp = stacky_server.do_list_usage_deletes(fake_request) + self.assertEqual(resp.status_code, 200) + resp_json = json.loads(resp.content) + self.assertEqual(len(resp_json), 2) + self.assertEqual(resp_json[0], ["UUID", "Launched At", + "Deleted At"]) + self.assertEqual(resp_json[1][0], INSTANCE_ID_1) + launch_time_str = dt.dt_from_decimal(usage.launched_at) + self.assertEqual(resp_json[1][1], str(launch_time_str)) + delete_time_str = dt.dt_from_decimal(usage.deleted_at) + self.assertEqual(resp_json[1][2], str(delete_time_str)) + self.mox.VerifyAll() + + def test_do_list_usage_deletes_with_instance(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'instance': INSTANCE_ID_1} + results = self.mox.CreateMockAnything() + models.InstanceDeletes.objects.filter(instance=INSTANCE_ID_1)\ + .AndReturn(results) + usage = self.mox.CreateMockAnything() + usage.instance = INSTANCE_ID_1 + usage.launched_at = utils.decimal_utc() + usage.deleted_at = usage.launched_at + 10 + results.__iter__().AndReturn([usage].__iter__()) + self.mox.ReplayAll() + + resp = stacky_server.do_list_usage_deletes(fake_request) + self.assertEqual(resp.status_code, 200) + resp_json = json.loads(resp.content) + self.assertEqual(len(resp_json), 2) + self.assertEqual(resp_json[0], ["UUID", "Launched At", + "Deleted At"]) + self.assertEqual(resp_json[1][0], INSTANCE_ID_1) + launch_time_str = dt.dt_from_decimal(usage.launched_at) + self.assertEqual(resp_json[1][1], str(launch_time_str)) + delete_time_str = dt.dt_from_decimal(usage.deleted_at) + self.assertEqual(resp_json[1][2], str(delete_time_str)) + self.mox.VerifyAll() + + def test_do_list_usage_exists(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {} + results = self.mox.CreateMockAnything() + models.InstanceExists.objects.all().AndReturn(results) + usage = self.mox.CreateMockAnything() + usage.instance = INSTANCE_ID_1 + usage.launched_at = utils.decimal_utc() + usage.deleted_at = usage.launched_at + 10 + usage.instance_type_id = 1 + usage.message_id = 'someid' + usage.status = 'pending' + results.__iter__().AndReturn([usage].__iter__()) + self.mox.ReplayAll() + + resp = stacky_server.do_list_usage_exists(fake_request) + self.assertEqual(resp.status_code, 200) + resp_json = json.loads(resp.content) + self.assertEqual(len(resp_json), 2) + self.assertEqual(resp_json[0], ["UUID", "Launched At", "Deleted At", + "Instance Type Id", "Message ID", + "Status"]) + self.assertEqual(resp_json[1][0], INSTANCE_ID_1) + launch_time_str = dt.dt_from_decimal(usage.launched_at) + self.assertEqual(resp_json[1][1], str(launch_time_str)) + delete_time_str = dt.dt_from_decimal(usage.deleted_at) + self.assertEqual(resp_json[1][2], str(delete_time_str)) + self.mox.VerifyAll() + + def test_do_list_usage_exists_with_instance(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'instance': INSTANCE_ID_1} + results = self.mox.CreateMockAnything() + models.InstanceExists.objects.filter(instance=INSTANCE_ID_1)\ + .AndReturn(results) + usage = self.mox.CreateMockAnything() + usage.instance = INSTANCE_ID_1 + usage.launched_at = utils.decimal_utc() + usage.deleted_at = usage.launched_at + 10 + usage.instance_type_id = 1 + usage.message_id = 'someid' + usage.status = 'pending' + results.__iter__().AndReturn([usage].__iter__()) + self.mox.ReplayAll() + + resp = stacky_server.do_list_usage_exists(fake_request) + self.assertEqual(resp.status_code, 200) + resp_json = json.loads(resp.content) + self.assertEqual(len(resp_json), 2) + self.assertEqual(resp_json[0], ["UUID", "Launched At", "Deleted At", + "Instance Type Id", "Message ID", + "Status"]) + self.assertEqual(resp_json[1][0], INSTANCE_ID_1) + launch_time_str = dt.dt_from_decimal(usage.launched_at) + self.assertEqual(resp_json[1][1], str(launch_time_str)) + delete_time_str = dt.dt_from_decimal(usage.deleted_at) + self.assertEqual(resp_json[1][2], str(delete_time_str)) + self.mox.VerifyAll() + From 6b8b2fb19716065a58bc293b4658b82a8aeab834 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Tue, 26 Feb 2013 15:39:33 -0500 Subject: [PATCH 3/5] Cutting do_kpi short if no raw's for provided tenant --- stacktach/stacky_server.py | 7 +++++++ tests/unit/test_stacky_server.py | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/stacktach/stacky_server.py b/stacktach/stacky_server.py index f5d5de2..2f64f52 100644 --- a/stacktach/stacky_server.py +++ b/stacktach/stacky_server.py @@ -268,6 +268,13 @@ def do_watch(request, deployment_id): def do_kpi(request, tenant_id=None): + if tenant_id: + if models.RawData.objects.filter(tenant=tenant_id).count() == 0: + results = [["Error", "Message"]] + message = "Could not find raws for tenant %s" % tenant_id + results.append(["NotFound", message]) + return rsp(results, 404) + yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1) yesterday = dt.dt_to_decimal(yesterday) trackers = models.RequestTracker.objects.select_related()\ diff --git a/tests/unit/test_stacky_server.py b/tests/unit/test_stacky_server.py index 4f78414..8e094c3 100644 --- a/tests/unit/test_stacky_server.py +++ b/tests/unit/test_stacky_server.py @@ -508,6 +508,9 @@ class StackyServerTestCase(unittest.TestCase): def test_do_kpi_with_tenant(self): fake_request = self.mox.CreateMockAnything() + objects = self.mox.CreateMockAnything() + models.RawData.objects.filter(tenant='55555').AndReturn(objects) + objects.count().AndReturn(1) results = self.mox.CreateMockAnything() models.RequestTracker.objects.select_related().AndReturn(results) results.exclude(last_timing=None).AndReturn(results) @@ -540,6 +543,9 @@ class StackyServerTestCase(unittest.TestCase): def test_do_kpi_with_tenant_no_match(self): fake_request = self.mox.CreateMockAnything() + objects = self.mox.CreateMockAnything() + models.RawData.objects.filter(tenant='55555').AndReturn(objects) + objects.count().AndReturn(1) results = self.mox.CreateMockAnything() models.RequestTracker.objects.select_related().AndReturn(results) results.exclude(last_timing=None).AndReturn(results) @@ -567,6 +573,24 @@ class StackyServerTestCase(unittest.TestCase): self.mox.VerifyAll() + def test_do_kpi_tenant_doesnt_exist(self): + fake_request = self.mox.CreateMockAnything() + objects = self.mox.CreateMockAnything() + models.RawData.objects.filter(tenant='55555').AndReturn(objects) + objects.count().AndReturn(0) + self.mox.ReplayAll() + + resp = stacky_server.do_kpi(fake_request, '55555') + self.assertEqual(resp.status_code, 404) + body = resp.content + body = json.loads(body) + self.assertEqual(len(body), 2) + self.assertEqual(body[0], ['Error', 'Message']) + msg = 'Could not find raws for tenant 55555' + self.assertEqual(body[1], ['NotFound', msg]) + + self.mox.VerifyAll() + def test_do_list_usage_launches(self): fake_request = self.mox.CreateMockAnything() fake_request.GET = {} From 07dcb5685a0586b30058d0d0d9706cfaad7ff6f1 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Tue, 26 Feb 2013 15:47:43 -0500 Subject: [PATCH 4/5] Cutting do_kpi short if no raw's for provided tenant --- tests/unit/test_stacktach_utils.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/unit/test_stacktach_utils.py diff --git a/tests/unit/test_stacktach_utils.py b/tests/unit/test_stacktach_utils.py new file mode 100644 index 0000000..54006a2 --- /dev/null +++ b/tests/unit/test_stacktach_utils.py @@ -0,0 +1 @@ +__author__ = 'andrewmelton' From adc7693eeb51eca1cc376e26bdee06775d9fde31 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Tue, 26 Feb 2013 16:42:32 -0500 Subject: [PATCH 5/5] Validating uuids on api calls --- stacktach/dbapi.py | 6 +- stacktach/stacky_server.py | 41 ++++++++++--- stacktach/utils.py | 18 ++++++ tests/unit/test_dbapi.py | 11 ++++ tests/unit/test_stacktach_utils.py | 45 +++++++++++++- tests/unit/test_stacky_server.py | 99 ++++++++++++++++++++++++++++-- tests/unit/utils.py | 14 ++--- 7 files changed, 214 insertions(+), 20 deletions(-) diff --git a/stacktach/dbapi.py b/stacktach/dbapi.py index 1b2cbb1..8d22b82 100644 --- a/stacktach/dbapi.py +++ b/stacktach/dbapi.py @@ -103,7 +103,11 @@ def _check_has_field(klass, field_name): def _get_filter_args(klass, request): filter_args = {} if 'instance' in request.GET: - filter_args['instance'] = request.GET['instance'] + uuid = request.GET['instance'] + filter_args['instance'] = uuid + if not utils.is_uuid_like(uuid): + msg = "%s is not uuid-like" % uuid + raise BadRequestException(msg) for (key, value) in request.GET.items(): diff --git a/stacktach/stacky_server.py b/stacktach/stacky_server.py index 2f64f52..e484ab1 100644 --- a/stacktach/stacky_server.py +++ b/stacktach/stacky_server.py @@ -8,7 +8,7 @@ from django.shortcuts import get_object_or_404 import datetime_to_decimal as dt import models -import views +import utils SECS_PER_HOUR = 60 * 60 SECS_PER_DAY = SECS_PER_HOUR * 24 @@ -76,6 +76,11 @@ def rsp(data, status=200): status=status) +def error_response(status, type, message): + results = [["Error", "Message"], [type, message]] + return rsp(results, status) + + def do_deployments(request): deployments = get_deployments() results = [["#", "Name"]] @@ -102,6 +107,10 @@ def do_hosts(request): def do_uuid(request): uuid = str(request.GET['uuid']) + if not utils.is_uuid_like(uuid): + msg = "%s is not uuid-like" % uuid + return error_response(400, 'Bad Request', msg) + related = models.RawData.objects.select_related().filter(instance=uuid)\ .order_by('when') results = [["#", "?", "When", "Deployment", "Event", "Host", "State", @@ -116,6 +125,10 @@ def do_uuid(request): def do_timings_uuid(request): uuid = request.GET['uuid'] + if not utils.is_uuid_like(uuid): + msg = "%s is not uuid-like" % uuid + return error_response(400, 'Bad Request', msg) + return rsp(get_timings_for_uuid(uuid)) @@ -167,6 +180,10 @@ def do_summary(request): def do_request(request): request_id = request.GET['request_id'] + if not utils.is_request_id_like(request_id): + msg = "%s is not request-id-like" % request_id + return error_response(400, 'Bad Request', msg) + events = models.RawData.objects.filter(request_id=request_id) \ .order_by('when') results = [["#", "?", "When", "Deployment", "Event", "Host", @@ -270,10 +287,8 @@ def do_watch(request, deployment_id): def do_kpi(request, tenant_id=None): if tenant_id: if models.RawData.objects.filter(tenant=tenant_id).count() == 0: - results = [["Error", "Message"]] message = "Could not find raws for tenant %s" % tenant_id - results.append(["NotFound", message]) - return rsp(results, 404) + return error_response(404, 'Not Found', message) yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1) yesterday = dt.dt_to_decimal(yesterday) @@ -297,7 +312,11 @@ def do_list_usage_launches(request): filter_args = {} if 'instance' in request.GET: - filter_args['instance'] = request.GET['instance'] + uuid = request.GET['instance'] + if not utils.is_uuid_like(uuid): + msg = "%s is not uuid-like" % uuid + return error_response(400, 'Bad Request', msg) + filter_args['instance'] = uuid if len(filter_args) > 0: launches = models.InstanceUsage.objects.filter(**filter_args) @@ -319,7 +338,11 @@ def do_list_usage_deletes(request): filter_args = {} if 'instance' in request.GET: - filter_args['instance'] = request.GET['instance'] + uuid = request.GET['instance'] + if not utils.is_uuid_like(uuid): + msg = "%s is not uuid-like" % uuid + return error_response(400, 'Bad Request', msg) + filter_args['instance'] = uuid if len(filter_args) > 0: deletes = models.InstanceDeletes.objects.filter(**filter_args) @@ -344,7 +367,11 @@ def do_list_usage_exists(request): filter_args = {} if 'instance' in request.GET: - filter_args['instance'] = request.GET['instance'] + uuid = request.GET['instance'] + if not utils.is_uuid_like(uuid): + msg = "%s is not uuid-like" % uuid + return error_response(400, 'Bad Request', msg) + filter_args['instance'] = uuid if len(filter_args) > 0: exists = models.InstanceExists.objects.filter(**filter_args) diff --git a/stacktach/utils.py b/stacktach/utils.py index e4f05a4..7145971 100644 --- a/stacktach/utils.py +++ b/stacktach/utils.py @@ -1,7 +1,9 @@ import datetime +import uuid from stacktach import datetime_to_decimal as dt + def str_time_to_unix(when): if 'T' in when: try: @@ -23,3 +25,19 @@ def str_time_to_unix(when): print "BAD DATE: ", e return dt.dt_to_decimal(when) + + +def is_uuid_like(val): + try: + converted = str(uuid.UUID(val)) + if '-' not in val: + converted = converted.replace('-', '') + return converted == val + except (TypeError, ValueError, AttributeError): + return False + + +def is_request_id_like(val): + if val[0:4] == 'req-': + val = val[4:] + return is_uuid_like(val) \ No newline at end of file diff --git a/tests/unit/test_dbapi.py b/tests/unit/test_dbapi.py index 3b5df59..5c891cf 100644 --- a/tests/unit/test_dbapi.py +++ b/tests/unit/test_dbapi.py @@ -47,6 +47,17 @@ class DBAPITestCase(unittest.TestCase): self.assertEquals(filter_args.get('launched_at__lte'), end_decimal) + def test_get_filter_args_bad_uuid(self): + fake_model = self.make_fake_model() + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'instance': 'obviouslybaduuid'} + self.mox.ReplayAll() + + self.assertRaises(dbapi.BadRequestException, dbapi._get_filter_args, + fake_model, fake_request) + + self.mox.VerifyAll() + def test_get_filter_args_bad_min_value(self): fake_request = self.mox.CreateMockAnything() fake_request.GET = {'launched_at_min': 'obviouslybaddatetime'} diff --git a/tests/unit/test_stacktach_utils.py b/tests/unit/test_stacktach_utils.py index 54006a2..765f37f 100644 --- a/tests/unit/test_stacktach_utils.py +++ b/tests/unit/test_stacktach_utils.py @@ -1 +1,44 @@ -__author__ = 'andrewmelton' +import unittest + +import mox + +from stacktach import utils as stacktach_utils +from utils import INSTANCE_ID_1 +from utils import MESSAGE_ID_1 +from utils import REQUEST_ID_1 + + +class StacktachUtilsTestCase(unittest.TestCase): + def setUp(self): + self.mox = mox.Mox() + + def tearDown(self): + self.mox.UnsetStubs() + + def test_is_uuid_like(self): + uuid = INSTANCE_ID_1 + self.assertTrue(stacktach_utils.is_uuid_like(uuid)) + + def test_is_uuid_like_no_dashes(self): + uuid = "08f685d963524dbc827196cc54bf14cd" + self.assertTrue(stacktach_utils.is_uuid_like(uuid)) + + def test_is_uuid_like_invalid(self): + uuid = "$-^&#$" + self.assertFalse(stacktach_utils.is_uuid_like(uuid)) + + def test_is_request_id_like_with_uuid(self): + uuid = MESSAGE_ID_1 + self.assertTrue(stacktach_utils.is_request_id_like(uuid)) + + def test_is_message_id_like_with_req_uuid(self): + uuid = REQUEST_ID_1 + self.assertTrue(stacktach_utils.is_request_id_like(uuid)) + + def test_is_message_id_like_invalid_req(self): + uuid = "req-$-^&#$" + self.assertFalse(stacktach_utils.is_request_id_like(uuid)) + + def test_is_message_id_like_invalid(self): + uuid = "$-^&#$" + self.assertFalse(stacktach_utils.is_request_id_like(uuid)) \ No newline at end of file diff --git a/tests/unit/test_stacky_server.py b/tests/unit/test_stacky_server.py index 8e094c3..015da55 100644 --- a/tests/unit/test_stacky_server.py +++ b/tests/unit/test_stacky_server.py @@ -10,6 +10,7 @@ from stacktach import stacky_server import utils from utils import INSTANCE_ID_1 from utils import INSTANCE_ID_2 +from utils import REQUEST_ID_1 class StackyServerTestCase(unittest.TestCase): @@ -58,7 +59,7 @@ class StackyServerTestCase(unittest.TestCase): raw.publisher = "api.example.com" raw.service = 'api' raw.host = 'example.com' - raw.request_id = 'req-1' + raw.request_id = REQUEST_ID_1 raw.json = '{"key": "value"}' return raw @@ -250,6 +251,36 @@ class StackyServerTestCase(unittest.TestCase): self.assertEqual(json_resp[1], body) self.mox.VerifyAll() + def test_do_uuid_bad_uuid(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'uuid': "obviouslybaduuid"} + self.mox.ReplayAll() + + resp = stacky_server.do_uuid(fake_request) + + self.assertEqual(resp.status_code, 400) + resp_json = json.loads(resp.content) + self.assertEqual(len(resp_json), 2) + self.assertEqual(resp_json[0], ['Error', 'Message']) + msg = 'obviouslybaduuid is not uuid-like' + self.assertEqual(resp_json[1], ['Bad Request', msg]) + self.mox.VerifyAll() + + def test_do_timings_uuid_bad_uuid(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'uuid': "obviouslybaduuid"} + self.mox.ReplayAll() + + resp = stacky_server.do_timings_uuid(fake_request) + + self.assertEqual(resp.status_code, 400) + resp_json = json.loads(resp.content) + self.assertEqual(len(resp_json), 2) + self.assertEqual(resp_json[0], ['Error', 'Message']) + msg = 'obviouslybaduuid is not uuid-like' + self.assertEqual(resp_json[1], ['Bad Request', msg]) + self.mox.VerifyAll() + def test_do_timings(self): fake_request = self.mox.CreateMockAnything() fake_request.GET = {'name': 'test.event'} @@ -314,10 +345,10 @@ class StackyServerTestCase(unittest.TestCase): def test_do_request(self): fake_request = self.mox.CreateMockAnything() - fake_request.GET = {'request_id': 'req-1'} + fake_request.GET = {'request_id': REQUEST_ID_1} raw = self._create_raw() results = self.mox.CreateMockAnything() - models.RawData.objects.filter(request_id='req-1').AndReturn(results) + models.RawData.objects.filter(request_id=REQUEST_ID_1).AndReturn(results) results.order_by('when').AndReturn(results) results.__iter__().AndReturn([raw].__iter__()) self.mox.ReplayAll() @@ -341,6 +372,21 @@ class StackyServerTestCase(unittest.TestCase): self.assertEqual(json_resp[1][8], None) self.mox.VerifyAll() + def test_do_request_bad_request_id(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'request_id': "obviouslybaduuid"} + self.mox.ReplayAll() + + resp = stacky_server.do_request(fake_request) + + self.assertEqual(resp.status_code, 400) + resp_json = json.loads(resp.content) + self.assertEqual(len(resp_json), 2) + self.assertEqual(resp_json[0], ['Error', 'Message']) + msg = 'obviouslybaduuid is not request-id-like' + self.assertEqual(resp_json[1], ['Bad Request', msg]) + self.mox.VerifyAll() + def _assert_on_show(self, values, raw): self.assertEqual(len(values), 12) self.assertEqual(values[0], ["Key", "Value"]) @@ -587,7 +633,7 @@ class StackyServerTestCase(unittest.TestCase): self.assertEqual(len(body), 2) self.assertEqual(body[0], ['Error', 'Message']) msg = 'Could not find raws for tenant 55555' - self.assertEqual(body[1], ['NotFound', msg]) + self.assertEqual(body[1], ['Not Found', msg]) self.mox.VerifyAll() @@ -642,6 +688,21 @@ class StackyServerTestCase(unittest.TestCase): self.mox.VerifyAll() + def test_do_list_usage_launches_bad_instance(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'instance': "obviouslybaduuid"} + self.mox.ReplayAll() + + resp = stacky_server.do_list_usage_launches(fake_request) + + self.assertEqual(resp.status_code, 400) + resp_json = json.loads(resp.content) + self.assertEqual(len(resp_json), 2) + self.assertEqual(resp_json[0], ['Error', 'Message']) + msg = 'obviouslybaduuid is not uuid-like' + self.assertEqual(resp_json[1], ['Bad Request', msg]) + self.mox.VerifyAll() + def test_do_list_usage_deletes(self): fake_request = self.mox.CreateMockAnything() fake_request.GET = {} @@ -693,6 +754,21 @@ class StackyServerTestCase(unittest.TestCase): self.assertEqual(resp_json[1][2], str(delete_time_str)) self.mox.VerifyAll() + def test_do_list_usage_deletes_bad_instance(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'instance': "obviouslybaduuid"} + self.mox.ReplayAll() + + resp = stacky_server.do_list_usage_deletes(fake_request) + + self.assertEqual(resp.status_code, 400) + resp_json = json.loads(resp.content) + self.assertEqual(len(resp_json), 2) + self.assertEqual(resp_json[0], ['Error', 'Message']) + msg = 'obviouslybaduuid is not uuid-like' + self.assertEqual(resp_json[1], ['Bad Request', msg]) + self.mox.VerifyAll() + def test_do_list_usage_exists(self): fake_request = self.mox.CreateMockAnything() fake_request.GET = {} @@ -752,3 +828,18 @@ class StackyServerTestCase(unittest.TestCase): self.assertEqual(resp_json[1][2], str(delete_time_str)) self.mox.VerifyAll() + def test_do_list_usage_exists_bad_instance(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'instance': "obviouslybaduuid"} + self.mox.ReplayAll() + + resp = stacky_server.do_list_usage_exists(fake_request) + + self.assertEqual(resp.status_code, 400) + resp_json = json.loads(resp.content) + self.assertEqual(len(resp_json), 2) + self.assertEqual(resp_json[0], ['Error', 'Message']) + msg = 'obviouslybaduuid is not uuid-like' + self.assertEqual(resp_json[1], ['Bad Request', msg]) + self.mox.VerifyAll() + diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 29591d2..dfc90bf 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -7,15 +7,15 @@ TENANT_ID_1 = 'testtenantid1' from stacktach import datetime_to_decimal as dt -INSTANCE_ID_1 = 'testinstanceid1' -INSTANCE_ID_2 = 'testinstanceid2' +INSTANCE_ID_1 = "08f685d9-6352-4dbc-8271-96cc54bf14cd" +INSTANCE_ID_2 = "515adf96-41d3-b86d-5467-e584edc61dab" -MESSAGE_ID_1 = 'testmessageid1' -MESSAGE_ID_2 = 'testmessageid2' +MESSAGE_ID_1 = "7f28f81b-29a2-43f2-9ba1-ccb3e53ab6c8" +MESSAGE_ID_2 = "4d596126-0f04-4329-865f-7b9a7bd69bcf" -REQUEST_ID_1 = 'testrequestid1' -REQUEST_ID_2 = 'testrequestid2' -REQUEST_ID_3 = 'testrequestid3' +REQUEST_ID_1 = 'req-611a4d70-9e47-4b27-a95e-27996cc40c06' +REQUEST_ID_2 = 'req-a951dec0-52ee-425d-9f56-d68bd1ad00ac' +REQUEST_ID_3 = 'req-039a33f7-5849-4406-8166-4db8cd085f52' def decimal_utc(t = datetime.datetime.utcnow()):