diff --git a/tripleo_common/image/image_uploader.py b/tripleo_common/image/image_uploader.py index c391076bd..966850a1d 100644 --- a/tripleo_common/image/image_uploader.py +++ b/tripleo_common/image/image_uploader.py @@ -62,12 +62,14 @@ CALL_TYPES = ( CALL_BLOB, CALL_UPLOAD, CALL_TAGS, + CALL_CATALOG ) = ( '/', '%(image)s/manifests/%(tag)s', '%(image)s/blobs/%(digest)s', '%(image)s/blobs/uploads/', '%(image)s/tags/list', + '/_catalog', ) MEDIA_TYPES = ( @@ -449,6 +451,55 @@ class BaseImageUploader(object): 'Layers': layers, } + def list(self, registry, session=None): + self.is_insecure_registry(registry) + url = self._image_to_url(registry) + catalog_url = self._build_url( + url, CALL_CATALOG + ) + catalog = session.get(catalog_url, timeout=30).json() + + tags_get_args = [] + for repo in catalog.get('repositories', []): + image = '%s/%s' % (registry, repo) + tags_get_args.append((self, image, session)) + p = futures.ThreadPoolExecutor(max_workers=16) + + images = [] + for image, tags in p.map(tags_for_image, tags_get_args): + if not tags: + continue + for tag in tags: + images.append('%s:%s' % (image, tag)) + return images + + def inspect(self, image, session=None): + image_url = self._image_to_url(image) + return self._inspect(image_url, session) + + @classmethod + @tenacity.retry( # Retry up to 5 times with jittered exponential backoff + reraise=True, + retry=tenacity.retry_if_exception_type( + requests.exceptions.RequestException + ), + wait=tenacity.wait_random_exponential(multiplier=1, max=10), + stop=tenacity.stop_after_attempt(5) + ) + def _tags_for_image(cls, image, session): + url = cls._image_to_url(image) + parts = { + 'image': url.path, + } + tags_url = cls._build_url( + url, CALL_TAGS % parts + ) + r = session.get(tags_url, timeout=30) + if r.status_code in (403, 404): + return image, [] + tags = r.json() + return image, tags.get('tags', []) + @classmethod def _image_to_url(cls, image): if '://' not in image: @@ -1573,3 +1624,8 @@ def discover_tag_from_inspect(args): fallback_tag = None return image, self._discover_tag_from_inspect( i, image, tag_from_label, fallback_tag) + + +def tags_for_image(args): + self, image, session = args + return self._tags_for_image(image, session) diff --git a/tripleo_common/tests/image/test_image_uploader.py b/tripleo_common/tests/image/test_image_uploader.py index e9002d840..3229e67de 100644 --- a/tripleo_common/tests/image/test_image_uploader.py +++ b/tripleo_common/tests/image/test_image_uploader.py @@ -612,6 +612,53 @@ class TestBaseImageUploader(base.TestCase): inspect(url1, session=session) ) + @mock.patch('concurrent.futures.ThreadPoolExecutor') + def test_list(self, mock_pool): + mock_pool.return_value.map.return_value = ( + ('localhost:8787/t/foo', ['a']), + ('localhost:8787/t/bar', ['b']), + ('localhost:8787/t/baz', ['c', 'd']), + ('localhost:8787/t/bink', []) + ) + session = mock.Mock() + session.get.return_value.json.return_value = { + 'repositories': ['t/foo', 't/bar', 't/baz', 't/bink'] + } + self.assertEqual( + [ + 'localhost:8787/t/foo:a', + 'localhost:8787/t/bar:b', + 'localhost:8787/t/baz:c', + 'localhost:8787/t/baz:d' + ], + self.uploader.list('localhost:8787', session=session) + ) + mock_pool.return_value.map.assert_called_once_with( + image_uploader.tags_for_image, + [ + (self.uploader, 'localhost:8787/t/foo', session), + (self.uploader, 'localhost:8787/t/bar', session), + (self.uploader, 'localhost:8787/t/baz', session), + (self.uploader, 'localhost:8787/t/bink', session) + ]) + + def test_tags_for_image(self): + session = mock.Mock() + r = mock.Mock() + r.status_code = 200 + r.json.return_value = {'tags': ['a', 'b', 'c']} + session.get.return_value = r + self.uploader.insecure_registries.add('localhost:8787') + url = 'docker://localhost:8787/t/foo' + image, tags = self.uploader._tags_for_image(url, session=session) + self.assertEqual(url, image) + self.assertEqual(['a', 'b', 'c'], tags) + + # test missing tags file + r.status_code = 404 + image, tags = self.uploader._tags_for_image(url, session=session) + self.assertEqual([], tags) + def test_image_tag_from_url(self): u = self.uploader self.assertEqual(