Glance scrubber should page thru images from registry

If the registry is limited in it's page size when returning the
list of images, the scrubber will only receive the first page.
This patch enables the scrubber to retrieve all the pages of
images.

The new tests are implemented with mock rather than mox. There
is another patch (https://review.openstack.org/#/c/111995) to
convert the existing code in test_scrubber to use mock.

Change-Id: I106f679be96bd39167337c4e27f4046f90d7d7ff
Closes-Bug: 1280343

glance/scrubber.py

@@ -344,6 +344,30 @@ class ScrubDBQueue(ScrubQueue):
         else:
             return False
 
+    def _get_images_page(self, marker):
+        filters = {'deleted': True,
+                   'is_public': 'none',
+                   'status': 'pending_delete'}
+
+        if marker:
+            return self.registry.get_images_detailed(filters=filters,
+                                                     marker=marker)
+        else:
+            return self.registry.get_images_detailed(filters=filters)
+
+    def _get_all_images(self):
+        """Generator to fetch all appropriate images, paging as needed."""
+
+        marker = None
+        while True:
+            images = self._get_images_page(marker)
+            if len(images) == 0:
+                break
+            marker = images[-1]['id']
+
+            for image in images:
+                yield image
+
     def _walk_all_locations(self, remove=False):
         """Returns a list of image id and location tuple from scrub queue.
 
@@ -351,11 +375,9 @@ class ScrubDBQueue(ScrubQueue):
 
         :retval a list of image id, location id and uri tuple from scrub queue
         """
-        filters = {'deleted': True,
-                   'is_public': 'none',
-                   'status': 'pending_delete'}
         ret = []
-        for image in self.registry.get_images_detailed(filters=filters):
+
+        for image in self._get_all_images():
             deleted_at = image.get('deleted_at')
             if not deleted_at:
                 continue

glance/tests/unit/test_scrubber.py

@@ -20,6 +20,7 @@ import uuid
 
 import eventlet
 import glance_store
+from mock import patch
 import mox
 from oslo.config import cfg
 
@@ -75,3 +76,76 @@ class TestScrubber(test_utils.BaseTestCase):
     def test_store_delete_notfound_exception(self):
         ex = exception.NotFound()
         self._scrubber_cleanup_with_store_delete_exception(ex)
+
+
+class TestScrubDBQueue(test_utils.BaseTestCase):
+
+    def setUp(self):
+        super(TestScrubDBQueue, self).setUp()
+
+    def tearDown(self):
+        super(TestScrubDBQueue, self).tearDown()
+
+    def _create_image_list(self, count):
+        images = []
+        for x in range(count):
+            images.append({'id': x})
+
+        return images
+
+    def test_get_all_images(self):
+        scrub_queue = scrubber.ScrubDBQueue()
+        images = self._create_image_list(15)
+        image_pager = ImagePager(images)
+
+        def make_get_images_detailed(pager):
+            def mock_get_images_detailed(filters, marker=None):
+                return pager()
+            return mock_get_images_detailed
+
+        with patch.object(scrub_queue.registry, 'get_images_detailed') \
+                as _mock_get_images_detailed:
+            _mock_get_images_detailed.side_effect = \
+                make_get_images_detailed(image_pager)
+            actual = list(scrub_queue._get_all_images())
+
+        self.assertEqual(images, actual)
+
+    def test_get_all_images_paged(self):
+        scrub_queue = scrubber.ScrubDBQueue()
+        images = self._create_image_list(15)
+        image_pager = ImagePager(images, page_size=4)
+
+        def make_get_images_detailed(pager):
+            def mock_get_images_detailed(filters, marker=None):
+                return pager()
+            return mock_get_images_detailed
+
+        with patch.object(scrub_queue.registry, 'get_images_detailed') \
+                as _mock_get_images_detailed:
+            _mock_get_images_detailed.side_effect = \
+                make_get_images_detailed(image_pager)
+            actual = list(scrub_queue._get_all_images())
+
+        self.assertEqual(images, actual)
+
+
+class ImagePager(object):
+    def __init__(self, images, page_size=0):
+        image_count = len(images)
+        if (page_size == 0) or (page_size > image_count):
+            page_size = image_count
+        self.image_batches = []
+        start = 0
+        l = len(images)
+        while start < l:
+            self.image_batches.append(images[start: start + page_size])
+            start += page_size
+            if (l - start) < page_size:
+                page_size = l - start
+
+    def __call__(self):
+        if len(self.image_batches) == 0:
+            return []
+        else:
+            return self.image_batches.pop(0)