Merge "Glance scrubber should page thru images from registry"

glance/scrubber.py

@@ -344,6 +344,30 @@ class ScrubDBQueue(ScrubQueue):
         else:
             return False
 
+    def _get_images_page(self, marker):
+        filters = {'deleted': True,
+                   'is_public': 'none',
+                   'status': 'pending_delete'}
+
+        if marker:
+            return self.registry.get_images_detailed(filters=filters,
+                                                     marker=marker)
+        else:
+            return self.registry.get_images_detailed(filters=filters)
+
+    def _get_all_images(self):
+        """Generator to fetch all appropriate images, paging as needed."""
+
+        marker = None
+        while True:
+            images = self._get_images_page(marker)
+            if len(images) == 0:
+                break
+            marker = images[-1]['id']
+
+            for image in images:
+                yield image
+
     def _walk_all_locations(self, remove=False):
         """Returns a list of image id and location tuple from scrub queue.
 
@@ -351,11 +375,9 @@ class ScrubDBQueue(ScrubQueue):
 
         :retval a list of image id, location id and uri tuple from scrub queue
         """
-        filters = {'deleted': True,
-                   'is_public': 'none',
-                   'status': 'pending_delete'}
         ret = []
-        for image in self.registry.get_images_detailed(filters=filters):
+
+        for image in self._get_all_images():
             deleted_at = image.get('deleted_at')
             if not deleted_at:
                 continue

glance/tests/unit/test_scrubber.py

@@ -20,6 +20,7 @@ import uuid
 
 import eventlet
 import glance_store
+from mock import patch
 import mox
 from oslo.config import cfg
 
@@ -75,3 +76,76 @@ class TestScrubber(test_utils.BaseTestCase):
     def test_store_delete_notfound_exception(self):
         ex = exception.NotFound()
         self._scrubber_cleanup_with_store_delete_exception(ex)
+
+
+class TestScrubDBQueue(test_utils.BaseTestCase):
+
+    def setUp(self):
+        super(TestScrubDBQueue, self).setUp()
+
+    def tearDown(self):
+        super(TestScrubDBQueue, self).tearDown()
+
+    def _create_image_list(self, count):
+        images = []
+        for x in range(count):
+            images.append({'id': x})
+
+        return images
+
+    def test_get_all_images(self):
+        scrub_queue = scrubber.ScrubDBQueue()
+        images = self._create_image_list(15)
+        image_pager = ImagePager(images)
+
+        def make_get_images_detailed(pager):
+            def mock_get_images_detailed(filters, marker=None):
+                return pager()
+            return mock_get_images_detailed
+
+        with patch.object(scrub_queue.registry, 'get_images_detailed') \
+                as _mock_get_images_detailed:
+            _mock_get_images_detailed.side_effect = \
+                make_get_images_detailed(image_pager)
+            actual = list(scrub_queue._get_all_images())
+
+        self.assertEqual(images, actual)
+
+    def test_get_all_images_paged(self):
+        scrub_queue = scrubber.ScrubDBQueue()
+        images = self._create_image_list(15)
+        image_pager = ImagePager(images, page_size=4)
+
+        def make_get_images_detailed(pager):
+            def mock_get_images_detailed(filters, marker=None):
+                return pager()
+            return mock_get_images_detailed
+
+        with patch.object(scrub_queue.registry, 'get_images_detailed') \
+                as _mock_get_images_detailed:
+            _mock_get_images_detailed.side_effect = \
+                make_get_images_detailed(image_pager)
+            actual = list(scrub_queue._get_all_images())
+
+        self.assertEqual(images, actual)
+
+
+class ImagePager(object):
+    def __init__(self, images, page_size=0):
+        image_count = len(images)
+        if (page_size == 0) or (page_size > image_count):
+            page_size = image_count
+        self.image_batches = []
+        start = 0
+        l = len(images)
+        while start < l:
+            self.image_batches.append(images[start: start + page_size])
+            start += page_size
+            if (l - start) < page_size:
+                page_size = l - start
+
+    def __call__(self):
+        if len(self.image_batches) == 0:
+            return []
+        else:
+            return self.image_batches.pop(0)