Add a method to list all images in a registry

The list method fetches the contents of /v2/_catalog then concurrently
fetches the tag for every image returned. This will allow for the
implementation of a "openstack tripleo container image list" command.
Our users have frequently asked how they can be sure that the
undercloud registry has been correctly populated with the images they
expect, and a simple list command will provide that.

This change also adds a public inspect method so that we can implement
an "openstack tripleo container image show" command.

Change-Id: I3b8e00ffdd94c234ea5c6a9aeb0990acec71c9e4
Blueprint: podman-support

tripleo_common/image/image_uploader.py

@@ -62,12 +62,14 @@ CALL_TYPES = (
     CALL_BLOB,
     CALL_UPLOAD,
     CALL_TAGS,
+    CALL_CATALOG
 ) = (
     '/',
     '%(image)s/manifests/%(tag)s',
     '%(image)s/blobs/%(digest)s',
     '%(image)s/blobs/uploads/',
     '%(image)s/tags/list',
+    '/_catalog',
 )
 
 MEDIA_TYPES = (
@@ -449,6 +451,55 @@ class BaseImageUploader(object):
             'Layers': layers,
         }
 
+    def list(self, registry, session=None):
+        self.is_insecure_registry(registry)
+        url = self._image_to_url(registry)
+        catalog_url = self._build_url(
+            url, CALL_CATALOG
+        )
+        catalog = session.get(catalog_url, timeout=30).json()
+
+        tags_get_args = []
+        for repo in catalog.get('repositories', []):
+            image = '%s/%s' % (registry, repo)
+            tags_get_args.append((self, image, session))
+        p = futures.ThreadPoolExecutor(max_workers=16)
+
+        images = []
+        for image, tags in p.map(tags_for_image, tags_get_args):
+            if not tags:
+                continue
+            for tag in tags:
+                images.append('%s:%s' % (image, tag))
+        return images
+
+    def inspect(self, image, session=None):
+        image_url = self._image_to_url(image)
+        return self._inspect(image_url, session)
+
+    @classmethod
+    @tenacity.retry(  # Retry up to 5 times with jittered exponential backoff
+        reraise=True,
+        retry=tenacity.retry_if_exception_type(
+            requests.exceptions.RequestException
+        ),
+        wait=tenacity.wait_random_exponential(multiplier=1, max=10),
+        stop=tenacity.stop_after_attempt(5)
+    )
+    def _tags_for_image(cls, image, session):
+        url = cls._image_to_url(image)
+        parts = {
+            'image': url.path,
+        }
+        tags_url = cls._build_url(
+            url, CALL_TAGS % parts
+        )
+        r = session.get(tags_url, timeout=30)
+        if r.status_code in (403, 404):
+            return image, []
+        tags = r.json()
+        return image, tags.get('tags', [])
+
     @classmethod
     def _image_to_url(cls, image):
         if '://' not in image:
@@ -1573,3 +1624,8 @@ def discover_tag_from_inspect(args):
         fallback_tag = None
     return image, self._discover_tag_from_inspect(
         i, image, tag_from_label, fallback_tag)
+
+
+def tags_for_image(args):
+    self, image, session = args
+    return self._tags_for_image(image, session)

tripleo_common/tests/image/test_image_uploader.py

@@ -612,6 +612,53 @@ class TestBaseImageUploader(base.TestCase):
             inspect(url1, session=session)
         )
 
+    @mock.patch('concurrent.futures.ThreadPoolExecutor')
+    def test_list(self, mock_pool):
+        mock_pool.return_value.map.return_value = (
+            ('localhost:8787/t/foo', ['a']),
+            ('localhost:8787/t/bar', ['b']),
+            ('localhost:8787/t/baz', ['c', 'd']),
+            ('localhost:8787/t/bink', [])
+        )
+        session = mock.Mock()
+        session.get.return_value.json.return_value = {
+            'repositories': ['t/foo', 't/bar', 't/baz', 't/bink']
+        }
+        self.assertEqual(
+            [
+                'localhost:8787/t/foo:a',
+                'localhost:8787/t/bar:b',
+                'localhost:8787/t/baz:c',
+                'localhost:8787/t/baz:d'
+            ],
+            self.uploader.list('localhost:8787', session=session)
+        )
+        mock_pool.return_value.map.assert_called_once_with(
+            image_uploader.tags_for_image,
+            [
+                (self.uploader, 'localhost:8787/t/foo', session),
+                (self.uploader, 'localhost:8787/t/bar', session),
+                (self.uploader, 'localhost:8787/t/baz', session),
+                (self.uploader, 'localhost:8787/t/bink', session)
+            ])
+
+    def test_tags_for_image(self):
+        session = mock.Mock()
+        r = mock.Mock()
+        r.status_code = 200
+        r.json.return_value = {'tags': ['a', 'b', 'c']}
+        session.get.return_value = r
+        self.uploader.insecure_registries.add('localhost:8787')
+        url = 'docker://localhost:8787/t/foo'
+        image, tags = self.uploader._tags_for_image(url, session=session)
+        self.assertEqual(url, image)
+        self.assertEqual(['a', 'b', 'c'], tags)
+
+        # test missing tags file
+        r.status_code = 404
+        image, tags = self.uploader._tags_for_image(url, session=session)
+        self.assertEqual([], tags)
+
     def test_image_tag_from_url(self):
         u = self.uploader
         self.assertEqual(