Adding in some tests, renaming aggregate functions

.gitignore

@@ -0,0 +1,4 @@
+.idea/
+
+*.pyc
+local_settings.py

settings.py

@@ -4,12 +4,16 @@ import os
 try:
     from local_settings import *
 
+    db_engine = STACKTACH_DB_ENGINE
     db_name = STACKTACH_DB_NAME
     db_host = STACKTACH_DB_HOST
     db_username = STACKTACH_DB_USERNAME
     db_password = STACKTACH_DB_PASSWORD
     install_dir = STACKTACH_INSTALL_DIR
 except ImportError:
+    db_engine = os.environ.get('STACKTACH_DB_ENGINE',
+                               'django.db.backends.mysql')
+    db_host = os.environ.get('STACKTACH_DB_HOST', "")
     db_name = os.environ['STACKTACH_DB_NAME']
     db_host = os.environ.get('STACKTACH_DB_HOST', "")
     db_username = os.environ['STACKTACH_DB_USERNAME']
@@ -27,7 +31,7 @@ MANAGERS = ADMINS
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.mysql',
+        'ENGINE': db_engine,
         'NAME': db_name,
         'USER': db_username,
         'PASSWORD': db_password,
@@ -88,7 +92,7 @@ STATICFILES_DIRS = (
     # Put strings here, like "/home/html/static" or "C:/www/django/static".
     # Always use forward slashes, even on Windows.
     # Don't forget to use absolute paths, not relative paths.
-    ['/root/stacktach/static/',]
+    [install_dir + 'static/',]
 )
 
 # List of finder classes that know how to find static files in

stacktach/datetime_to_decimal.py

@@ -17,13 +17,3 @@ def dt_from_decimal(dec):
 
     daittyme = datetime.datetime.utcfromtimestamp(integer)
     return daittyme.replace(microsecond=micro)
-
-
-if __name__ == '__main__':
-    now = datetime.datetime.utcnow()
-    d = dt_to_decimal(now)
-    daittyme = dt_from_decimal(d)
-    print repr(now)
-    print repr(d)
-    print repr(daittyme)
-    assert(now == daittyme)

stacktach/models.py

@@ -54,7 +54,7 @@ class RawData(models.Model):
                                 blank=True, db_index=True)
 
     def __repr__(self):
-        return self.event
+        return "%s %s %s" % (self.event, self.instance, self.state)
 
 
 class Lifecycle(models.Model):

stacktach/tests.py

@@ -0,0 +1,381 @@
+import datetime
+import decimal
+import time
+
+from django.utils import unittest
+
+import datetime_to_decimal
+from models import *
+import views
+
+INSTANCE_ID_1 = 'testinstanceid1'
+INSTANCE_ID_2 = 'testinstanceid2'
+
+REQUEST_ID_1 = 'testrequestid1'
+REQUEST_ID_2 = 'testrequestid2'
+
+
+class DatetimeToDecimalTestCase(unittest.TestCase):
+
+    def test_datetime_to_and_from_decimal(self):
+        now = datetime.datetime.utcnow()
+        d = datetime_to_decimal.dt_to_decimal(now)
+        daittyme = datetime_to_decimal.dt_from_decimal(d)
+        self.assertEqual(now, daittyme)
+
+    def test_datetime_to_decimal(self):
+        expected_decimal = decimal.Decimal('1356093296.123')
+        utc_datetime = datetime.datetime.utcfromtimestamp(expected_decimal)
+        actual_decimal = datetime_to_decimal.dt_to_decimal(utc_datetime)
+        self.assertEqual(actual_decimal, expected_decimal)
+
+    def test_decimal_to_datetime(self):
+        expected_decimal = decimal.Decimal('1356093296.123')
+        expected_datetime = datetime.datetime.utcfromtimestamp(expected_decimal)
+        actual_datetime = datetime_to_decimal.dt_from_decimal(expected_decimal)
+        self.assertEqual(actual_datetime, expected_datetime)
+
+
+class ViewsUtilsTestCase(unittest.TestCase):
+
+    def test_srt_time_to_unix(self):
+        unix = views.str_time_to_unix('2012-12-21 12:34:56.123')
+        self.assertEqual(unix, decimal.Decimal('1356093296.123'))
+
+
+class ViewsLifecycleTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.deployment = Deployment(name='TestDeployment')
+        self.deployment.save()
+        when1 = views.str_time_to_unix('2012-12-21 12:34:50.123')
+        when2 = views.str_time_to_unix('2012-12-21 12:34:56.123')
+        when3 = views.str_time_to_unix('2012-12-21 12:36:56.124')
+        self.update_raw = self.create_raw(self.deployment, when1,
+                                          'compute.instance.update',
+                                          host='api')
+        self.start_raw = self.create_raw(self.deployment, when2,
+                                         'compute.instance.reboot.start')
+        self.end_raw = self.create_raw(self.deployment, when3,
+                                       'compute.instance.reboot.end',
+                                       old_task='reboot')
+
+    def create_raw(self, deployment, when, event, instance=INSTANCE_ID_1,
+                request_id=REQUEST_ID_1, state='active', old_task='',
+                host='compute'):
+        raw_values  = {
+            'deployment': deployment,
+            'host': host,
+            'state': state,
+            'old_task': old_task,
+            'when': when,
+            'event': event,
+            'instance': instance,
+            'request_id': request_id,
+            }
+        raw = RawData(**raw_values)
+        raw.save()
+        return raw
+
+    def tearDown(self):
+        Deployment.objects.all().delete()
+        RawData.objects.all().delete()
+        Lifecycle.objects.all().delete()
+        Timing.objects.all().delete()
+        RequestTracker.objects.all().delete()
+
+
+    def assertOnLifecycle(self, lifecycle, instance, last_raw):
+        self.assertEqual(lifecycle.instance, instance)
+        self.assertEqual(lifecycle.last_raw.id, last_raw.id)
+        self.assertEqual(lifecycle.last_state, last_raw.state)
+        self.assertEqual(lifecycle.last_task_state, last_raw.old_task)
+
+    def assertOnTiming(self, timing, lifecycle, start_raw, end_raw, diff):
+        self.assertEqual(timing.lifecycle.id, lifecycle.id)
+        self.assertEqual(timing.start_raw.id, start_raw.id)
+        self.assertEqual(timing.end_raw.id, end_raw.id)
+        self.assertEqual(timing.start_when, start_raw.when)
+        self.assertEqual(timing.end_when, end_raw.when)
+        self.assertEqual(timing.diff, decimal.Decimal(diff))
+
+    def assertOnTracker(self, tracker, request_id, lifecycle, start, diff=None):
+        self.assertEqual(tracker.request_id, request_id)
+        self.assertEqual(tracker.lifecycle.id, lifecycle.id)
+        self.assertEqual(tracker.start, start)
+        if diff:
+            self.assertEqual(tracker.duration, diff)
+
+    def test_aggregate_lifecycle_and_timing(self):
+        views.aggregate_lifecycle(self.update_raw)
+        views.aggregate_lifecycle(self.start_raw)
+
+        lifecycles = Lifecycle.objects.select_related()\
+                                      .filter(instance=INSTANCE_ID_1)
+        self.assertEqual(len(lifecycles), 1)
+        lifecycle = lifecycles[0]
+        self.assertOnLifecycle(lifecycle, INSTANCE_ID_1, self.start_raw)
+
+        views.aggregate_lifecycle(self.end_raw)
+
+        lifecycles = Lifecycle.objects.select_related()\
+                                      .filter(instance=INSTANCE_ID_1)
+        self.assertEqual(len(lifecycles), 1)
+        lifecycle = lifecycles[0]
+        self.assertOnLifecycle(lifecycle, INSTANCE_ID_1, self.end_raw)
+
+        timings = Timing.objects.select_related()\
+                                .filter(lifecycle=lifecycle)
+        self.assertEqual(len(lifecycles), 1)
+        timing = timings[0]
+        expected_diff = self.end_raw.when - self.start_raw.when
+        self.assertOnTiming(timing, lifecycle, self.start_raw, self.end_raw,
+                            expected_diff)
+
+    def test_multiple_instance_lifecycles(self):
+        when1 = views.str_time_to_unix('2012-12-21 13:32:50.123')
+        when2 = views.str_time_to_unix('2012-12-21 13:34:50.123')
+        when3 = views.str_time_to_unix('2012-12-21 13:37:50.124')
+        update_raw2 = self.create_raw(self.deployment, when1,
+                                      'compute.instance.update',
+                                      instance=INSTANCE_ID_2,
+                                      request_id=REQUEST_ID_2,
+                                      host='api')
+        start_raw2 = self.create_raw(self.deployment, when2,
+                                     'compute.instance.resize.start',
+                                     instance=INSTANCE_ID_2,
+                                     request_id=REQUEST_ID_2)
+        end_raw2 = self.create_raw(self.deployment, when3,
+                                   'compute.instance.resize.end',
+                                   old_task='resize',
+                                   instance=INSTANCE_ID_2,
+                                   request_id=REQUEST_ID_2)
+
+        views.aggregate_lifecycle(self.update_raw)
+        views.aggregate_lifecycle(self.start_raw)
+        views.aggregate_lifecycle(update_raw2)
+        views.aggregate_lifecycle(start_raw2)
+
+        lifecycles = Lifecycle.objects.all().order_by('id')
+        self.assertEqual(len(lifecycles), 2)
+        lifecycle1 = lifecycles[0]
+        self.assertOnLifecycle(lifecycle1, INSTANCE_ID_1, self.start_raw)
+        lifecycle2 = lifecycles[1]
+        self.assertOnLifecycle(lifecycle2, INSTANCE_ID_2, start_raw2)
+
+        views.aggregate_lifecycle(end_raw2)
+        views.aggregate_lifecycle(self.end_raw)
+
+        lifecycles = Lifecycle.objects.all().order_by('id')
+        self.assertEqual(len(lifecycles), 2)
+        lifecycle1 = lifecycles[0]
+        self.assertOnLifecycle(lifecycle1, INSTANCE_ID_1, self.end_raw)
+        lifecycle2 = lifecycles[1]
+        self.assertOnLifecycle(lifecycle2, INSTANCE_ID_2, end_raw2)
+
+        timings = Timing.objects.all().order_by('id')
+        self.assertEqual(len(timings), 2)
+        timing1 = timings[0]
+        expected_diff1 = self.end_raw.when - self.start_raw.when
+        self.assertOnTiming(timing1, lifecycle1, self.start_raw, self.end_raw,
+                            expected_diff1)
+        expected_diff2 = end_raw2.when - start_raw2.when
+        timing2 = timings[1]
+        self.assertOnTiming(timing2, lifecycle2, start_raw2, end_raw2,
+            expected_diff2)
+
+
+    def test_same_instance_multiple_timings(self):
+        when1 = views.str_time_to_unix('2012-12-21 13:32:50.123')
+        when2 = views.str_time_to_unix('2012-12-21 13:34:50.123')
+        when3 = views.str_time_to_unix('2012-12-21 13:37:50.124')
+        update_raw2 = self.create_raw(self.deployment, when1,
+                                      'compute.instance.update',
+                                      request_id=REQUEST_ID_2,
+                                      host='api')
+        start_raw2 = self.create_raw(self.deployment, when2,
+                                     'compute.instance.resize.start',
+                                     request_id=REQUEST_ID_2)
+        end_raw2 = self.create_raw(self.deployment, when3,
+                                   'compute.instance.resize.end',
+                                   old_task='resize',
+                                   request_id=REQUEST_ID_2)
+
+        # First action started
+        views.aggregate_lifecycle(self.update_raw)
+        views.aggregate_lifecycle(self.start_raw)
+        # Second action started, first end is late
+        views.aggregate_lifecycle(update_raw2)
+        views.aggregate_lifecycle(start_raw2)
+        # Finally get first end
+        views.aggregate_lifecycle(self.end_raw)
+        # Second end
+        views.aggregate_lifecycle(end_raw2)
+
+        lifecycles = Lifecycle.objects.select_related()\
+                                      .filter(instance=INSTANCE_ID_1)
+        self.assertEqual(len(lifecycles), 1)
+        lifecycle1 = lifecycles[0]
+        self.assertOnLifecycle(lifecycle1, INSTANCE_ID_1, end_raw2)
+
+        timings = Timing.objects.all().order_by('id')
+        self.assertEqual(len(timings), 2)
+        timing1 = timings[0]
+        expected_diff1 = self.end_raw.when - self.start_raw.when
+        self.assertOnTiming(timing1, lifecycle1, self.start_raw, self.end_raw,
+                            expected_diff1)
+        expected_diff2 = end_raw2.when - start_raw2.when
+        timing2 = timings[1]
+        self.assertOnTiming(timing2, lifecycle1, start_raw2, end_raw2,
+                            expected_diff2)
+
+    def test_aggregate_lifecycle_and_kpi(self):
+        views.aggregate_lifecycle(self.update_raw)
+
+        lifecycles = Lifecycle.objects.select_related()\
+                                      .filter(instance=INSTANCE_ID_1)
+        self.assertEqual(len(lifecycles), 1)
+        lifecycle = lifecycles[0]
+        self.assertOnLifecycle(lifecycle, INSTANCE_ID_1, self.update_raw)
+
+        trackers = RequestTracker.objects.filter(request_id=REQUEST_ID_1)
+        self.assertEqual(len(trackers), 1)
+        tracker = trackers[0]
+        self.assertOnTracker(tracker, REQUEST_ID_1, lifecycle,
+                             self.update_raw.when)
+
+        views.aggregate_lifecycle(self.start_raw)
+        views.aggregate_lifecycle(self.end_raw)
+
+        trackers = RequestTracker.objects.filter(request_id=REQUEST_ID_1)
+        self.assertEqual(len(trackers), 1)
+        tracker = trackers[0]
+        expected_diff = self.end_raw.when-self.update_raw.when
+        self.assertOnTracker(tracker, REQUEST_ID_1, lifecycle,
+                             self.update_raw.when, expected_diff)
+
+    def test_multiple_instance_kpi(self):
+        when1 = views.str_time_to_unix('2012-12-21 13:32:50.123')
+        when2 = views.str_time_to_unix('2012-12-21 13:34:50.123')
+        when3 = views.str_time_to_unix('2012-12-21 13:37:50.124')
+        update_raw2 = self.create_raw(self.deployment, when1,
+                                      'compute.instance.update',
+                                      instance=INSTANCE_ID_2,
+                                      request_id=REQUEST_ID_2,
+                                      host='api')
+        start_raw2 = self.create_raw(self.deployment, when2,
+                                     'compute.instance.resize.start',
+                                     instance=INSTANCE_ID_2,
+                                     request_id=REQUEST_ID_2)
+        end_raw2 = self.create_raw(self.deployment, when3,
+                                   'compute.instance.resize.end',
+                                   instance=INSTANCE_ID_2,
+                                   old_task='resize',
+                                   request_id=REQUEST_ID_2)
+
+        views.aggregate_lifecycle(self.update_raw)
+        views.aggregate_lifecycle(self.start_raw)
+        views.aggregate_lifecycle(self.end_raw)
+        views.aggregate_lifecycle(update_raw2)
+        views.aggregate_lifecycle(start_raw2)
+        views.aggregate_lifecycle(end_raw2)
+
+        lifecycles = Lifecycle.objects.all().order_by('id')
+        self.assertEqual(len(lifecycles), 2)
+        lifecycle1 = lifecycles[0]
+        self.assertOnLifecycle(lifecycle1, INSTANCE_ID_1, self.end_raw)
+        lifecycle2 = lifecycles[1]
+        self.assertOnLifecycle(lifecycle2, INSTANCE_ID_2, end_raw2)
+
+        trackers = RequestTracker.objects.all().order_by('id')
+        self.assertEqual(len(trackers), 2)
+        tracker1 = trackers[0]
+        expected_diff = self.end_raw.when-self.update_raw.when
+        self.assertOnTracker(tracker1, REQUEST_ID_1, lifecycle1,
+                             self.update_raw.when, expected_diff)
+        tracker2 = trackers[1]
+        expected_diff2 = end_raw2.when-update_raw2.when
+        self.assertOnTracker(tracker2, REQUEST_ID_2, lifecycle2,
+                             update_raw2.when, expected_diff2)
+
+    def test_single_instance_multiple_kpi(self):
+        when1 = views.str_time_to_unix('2012-12-21 13:32:50.123')
+        when2 = views.str_time_to_unix('2012-12-21 13:34:50.123')
+        when3 = views.str_time_to_unix('2012-12-21 13:37:50.124')
+        update_raw2 = self.create_raw(self.deployment, when1,
+            'compute.instance.update',
+            request_id=REQUEST_ID_2,
+            host='api')
+        start_raw2 = self.create_raw(self.deployment, when2,
+            'compute.instance.resize.start',
+            request_id=REQUEST_ID_2)
+        end_raw2 = self.create_raw(self.deployment, when3,
+            'compute.instance.resize.end',
+            old_task='resize',
+            request_id=REQUEST_ID_2)
+
+        views.aggregate_lifecycle(self.update_raw)
+        views.aggregate_lifecycle(self.start_raw)
+        views.aggregate_lifecycle(self.end_raw)
+        views.aggregate_lifecycle(update_raw2)
+        views.aggregate_lifecycle(start_raw2)
+        views.aggregate_lifecycle(end_raw2)
+
+        lifecycles = Lifecycle.objects.all().order_by('id')
+        self.assertEqual(len(lifecycles), 1)
+        lifecycle1 = lifecycles[0]
+        self.assertOnLifecycle(lifecycle1, INSTANCE_ID_1, end_raw2)
+
+        trackers = RequestTracker.objects.all().order_by('id')
+        self.assertEqual(len(trackers), 2)
+        tracker1 = trackers[0]
+        expected_diff1 = self.end_raw.when-self.update_raw.when
+        self.assertOnTracker(tracker1, REQUEST_ID_1, lifecycle1,
+                             self.update_raw.when, expected_diff1)
+        tracker2 = trackers[1]
+        expected_diff2 = end_raw2.when-update_raw2.when
+        self.assertOnTracker(tracker2, REQUEST_ID_2, lifecycle1,
+                             update_raw2.when, expected_diff2)
+
+    def test_single_instance_multiple_kpi_out_of_order(self):
+        when1 = views.str_time_to_unix('2012-12-21 13:32:50.123')
+        when2 = views.str_time_to_unix('2012-12-21 13:34:50.123')
+        when3 = views.str_time_to_unix('2012-12-21 13:37:50.124')
+        update_raw2 = self.create_raw(self.deployment, when1,
+                                      'compute.instance.update',
+                                      request_id=REQUEST_ID_2,
+                                      host='api')
+        start_raw2 = self.create_raw(self.deployment, when2,
+                                     'compute.instance.resize.start',
+                                     request_id=REQUEST_ID_2)
+        end_raw2 = self.create_raw(self.deployment, when3,
+                                   'compute.instance.resize.end',
+                                   old_task='resize',
+                                   request_id=REQUEST_ID_2)
+
+        # First action started
+        views.aggregate_lifecycle(self.update_raw)
+        views.aggregate_lifecycle(self.start_raw)
+        # Second action started, first end is late
+        views.aggregate_lifecycle(update_raw2)
+        views.aggregate_lifecycle(start_raw2)
+        # Finally get first end
+        views.aggregate_lifecycle(self.end_raw)
+        # Second end
+        views.aggregate_lifecycle(end_raw2)
+
+        lifecycles = Lifecycle.objects.all().order_by('id')
+        self.assertEqual(len(lifecycles), 1)
+        lifecycle1 = lifecycles[0]
+        self.assertOnLifecycle(lifecycle1, INSTANCE_ID_1, end_raw2)
+
+        trackers = RequestTracker.objects.all().order_by('id')
+        self.assertEqual(len(trackers), 2)
+        tracker1 = trackers[0]
+        expected_diff1 = self.end_raw.when-self.update_raw.when
+        self.assertOnTracker(tracker1, REQUEST_ID_1, lifecycle1,
+                             self.update_raw.when, expected_diff1)
+        tracker2 = trackers[1]
+        expected_diff2 = end_raw2.when-update_raw2.when
+        self.assertOnTracker(tracker2, REQUEST_ID_2, lifecycle1,
+                             update_raw2.when, expected_diff2)

stacktach/views.py

@@ -113,7 +113,7 @@ def update_kpi(lifecycle, timing, raw):
     tracker.save()
 
 
-def aggregate(raw):
+def aggregate_lifecycle(raw):
     """Roll up the raw event into a Lifecycle object
     and a bunch of Timing objects.
 
@@ -211,10 +211,10 @@ INSTANCE_EVENT = {
     'exists': 'compute.instance.exists',
 }
 
-def process_for_usage(raw):
+def aggregate_usage(raw):
     if not raw.instance:
         return
-    
+
     if raw.event == INSTANCE_EVENT['create_start'] or \
             raw.event == INSTANCE_EVENT['resize_prep_start'] or\
             raw.event == INSTANCE_EVENT['resize_revert_start']:
@@ -239,10 +239,10 @@ def _process_usage_for_new_launch(raw):
     values = {}
     values['instance'] = payload['instance_id']
     values['request_id'] = notif[1]['_context_request_id']
-    
+
     if raw.event == INSTANCE_EVENT['create_start']:
         values['instance_type_id'] = payload['instance_type_id']
-    
+
     usage = models.InstanceUsage(**values)
     usage.save()
 
@@ -259,10 +259,10 @@ def _process_usage_for_updates(raw):
             raw.event == INSTANCE_EVENT['resize_finish_end'] or\
             raw.event == INSTANCE_EVENT['resize_revert_end']:
         instance.launched_at = str_time_to_unix(payload['launched_at'])
-    
+
     if raw.event == INSTANCE_EVENT['resize_revert_end']:
         instance.instance_type_id = payload['instance_type_id']
-    elif raw.event == INSTANCE_EVENT['resize_prep_end']: 
+    elif raw.event == INSTANCE_EVENT['resize_prep_end']:
         instance.instance_type_id = payload['new_instance_type_id']
 
     instance.save()
@@ -293,14 +293,14 @@ def _process_exists(raw):
     values['instance'] = instance_id
     values['launched_at'] = launched_at
     values['instance_type_id'] = payload['instance_type_id']
-    
+
     values['usage'] = usage
     values['raw'] = raw
     deleted_at = payload.get('deleted_at')
     if deleted_at and deleted_at != '':
         deleted_at = str_time_to_unix(deleted_at)
         values['deleted_at'] = deleted_at
-    
+
     exists = models.InstanceExists(**values)
     exists.save()
 
@@ -348,8 +348,8 @@ def process_raw_data(deployment, args, json_args):
         record = models.RawData(**values)
         record.save()
 
-        aggregate(record)
+        aggregate_lifecycle(record)
-        process_for_usage(record)
+        aggregate_usage(record)
     return record