improved the container listings for the stats processor

swift/stats/log_processor.py

@@ -113,48 +113,35 @@ class LogProcessor(object):
                               listing_filter=None):
         '''
         Get a container listing, filtered by start_date, end_date, and
-        listing_filter. Dates, if given, should be in YYYYMMDDHH format
+        listing_filter. Dates, if given, must be in YYYYMMDDHH format
         '''
         search_key = None
         if start_date is not None:
-            date_parts = []
             try:
-                year, start_date = start_date[:4], start_date[4:]
+                parsed_date = time.strptime(start_date, '%Y%m%d%H')
-                if year:
+            except ValueError:
-                    date_parts.append(year)
-                    month, start_date = start_date[:2], start_date[2:]
-                    if month:
-                        date_parts.append(month)
-                        day, start_date = start_date[:2], start_date[2:]
-                        if day:
-                            date_parts.append(day)
-                            hour, start_date = start_date[:2], start_date[2:]
-                            if hour:
-                                date_parts.append(hour)
-            except IndexError:
                 pass
             else:
-                search_key = '/'.join(date_parts)
+                year = '%04d' % parsed_date.tm_year
+                month = '%02d' % parsed_date.tm_mon
+                day = '%02d' % parsed_date.tm_mday
+                hour = '%02d' % parsed_date.tm_hour
+                search_key = '/'.join([year, month, day, hour])
         end_key = None
         if end_date is not None:
-            date_parts = []
             try:
-                year, end_date = end_date[:4], end_date[4:]
+                parsed_date = time.strptime(end_date, '%Y%m%d%H')
-                if year:
+            except ValueError:
-                    date_parts.append(year)
-                    month, end_date = end_date[:2], end_date[2:]
-                    if month:
-                        date_parts.append(month)
-                        day, end_date = end_date[:2], end_date[2:]
-                        if day:
-                            date_parts.append(day)
-                            hour, end_date = end_date[:2], end_date[2:]
-                            if hour:
-                                date_parts.append(hour)
-            except IndexError:
                 pass
             else:
-                end_key = '/'.join(date_parts)
+                year = '%04d' % parsed_date.tm_year
+                month = '%02d' % parsed_date.tm_mon
+                day = '%02d' % parsed_date.tm_mday
+                hour = '%02d' % parsed_date.tm_hour
+                # Since the end_marker filters by <=, we need to add something
+                # to then end_key to make sure we get all the data under the
+                # last hour. Adding '/\x7f' should be all inclusive.
+                end_key = '/'.join([year, month, day, hour]) + '/\x7f'
         container_listing = self.internal_proxy.get_container_list(
                                     swift_account,
                                     container_name,

test/unit/stats/test_log_processor.py

@@ -37,9 +37,10 @@ class DumbInternalProxy(object):
                            end_marker=None):
         n = '2010/03/14/13/obj1'
         if marker is None or n > marker:
-            if end_marker and n <= end_marker:
+            if end_marker:
+                if n <= end_marker:
                     return [{'name': n}]
-            elif end_marker:
+                else:
                     return []
             return [{'name': n}]
         return []
@@ -178,6 +179,10 @@ use = egg:swift#proxy
                                             end_date='2010031412')
         expected = []
         self.assertEquals(result, expected)
+        result = p.get_container_listing('a', 'foo', start_date='2010031412',
+                                            end_date='2010031413')
+        expected = ['2010/03/14/13/obj1']
+        self.assertEquals(result, expected)
 
     def test_get_object_data(self):
         p = log_processor.LogProcessor(self.proxy_config, DumbLogger())
@@ -212,7 +217,6 @@ use = egg:swift#proxy
         p = log_processor.LogProcessor(self.proxy_config, DumbLogger())
         result = p.generate_keylist_mapping()
         expected = {}
-        print p.plugins
         self.assertEquals(result, expected)
 
     def test_generate_keylist_mapping_with_dummy_plugins(self):