diff --git a/tripleo_common/image/image_uploader.py b/tripleo_common/image/image_uploader.py index 8bef36f2b..f49b9ccd0 100644 --- a/tripleo_common/image/image_uploader.py +++ b/tripleo_common/image/image_uploader.py @@ -77,7 +77,8 @@ class ImageUploadManager(BaseImageManager): """ def __init__(self, config_files=None, - dry_run=False, cleanup=CLEANUP_FULL): + dry_run=False, cleanup=CLEANUP_FULL, + mirrors=None): if config_files is None: config_files = [] super(ImageUploadManager, self).__init__(config_files) @@ -87,13 +88,14 @@ class ImageUploadManager(BaseImageManager): } self.dry_run = dry_run self.cleanup = cleanup + self.mirrors = mirrors def discover_image_tag(self, image, tag_from_label=None, username=None, password=None): uploader = self.uploader(DEFAULT_UPLOADER) return uploader.discover_image_tag( image, tag_from_label=tag_from_label, - username=username, password=password) + username=username, password=password, mirrors=self.mirrors) def uploader(self, uploader): if uploader not in self.uploaders: @@ -136,7 +138,7 @@ class ImageUploadManager(BaseImageManager): task = UploadTask( image_name, pull_source, push_destination, append_tag, modify_role, modify_vars, self.dry_run, - self.cleanup) + self.cleanup, self.mirrors) uploader.add_upload_task(task) for uploader in self.uploaders.values(): @@ -221,13 +223,14 @@ class BaseImageUploader(object): 'Modifying image %s failed' % target_image) @classmethod - def _images_match(cls, image1, image2, session1=None): + def _images_match(cls, image1, image2, session1=None, mirrors=None): try: - image1_digest = cls._image_digest(image1, session=session1) + image1_digest = cls._image_digest(image1, session=session1, + mirrors=mirrors) except Exception: return False try: - image2_digest = cls._image_digest(image2) + image2_digest = cls._image_digest(image2, mirrors=mirrors) except Exception: return False @@ -237,22 +240,22 @@ class BaseImageUploader(object): return image1_digest == image2_digest @classmethod - def _image_digest(cls, image, session=None): + def _image_digest(cls, image, session=None, mirrors=None): image_url = cls._image_to_url(image) insecure = image_url.netloc in cls.insecure_registries - i = cls._inspect(image_url, insecure, session) + i = cls._inspect(image_url, insecure, session, mirrors=mirrors) return i.get('Digest') @classmethod - def _image_labels(cls, image_url, insecure, session=None): - i = cls._inspect(image_url, insecure, session) + def _image_labels(cls, image_url, insecure, session=None, mirrors=None): + i = cls._inspect(image_url, insecure, session, mirrors=mirrors) return i.get('Labels', {}) or {} @classmethod - def _image_exists(cls, image, session=None): + def _image_exists(cls, image, session=None, mirrors=None): try: cls._image_digest( - image, session=session) + image, session=session, mirrors=mirrors) except ImageNotFoundException: return False else: @@ -268,15 +271,11 @@ class BaseImageUploader(object): stop=tenacity.stop_after_attempt(5) ) def authenticate(cls, image_url, username=None, password=None, - insecure=False): - image_url = cls._fix_dockerio_url(image_url) - netloc = image_url.netloc - if insecure: - scheme = 'http' - else: - scheme = 'https' + insecure=False, mirrors=None): image, tag = image_url.path.split(':') - url = '%s://%s/v2/' % (scheme, netloc) + url = cls._build_url(image_url, path='/', + insecure=insecure, + mirrors=mirrors) session = requests.Session() r = session.get(url, timeout=30) LOG.debug('%s status code %s' % (url, r.status_code)) @@ -308,14 +307,19 @@ class BaseImageUploader(object): return session @classmethod - def _fix_dockerio_url(cls, url): - one = 'docker.io' - two = 'registry-1.docker.io' - if url.netloc != one: - return url - return parse.ParseResult(url.scheme, two, - url.path, url.params, - url.query, url.fragment) + def _build_url(cls, url, path, insecure=False, mirrors=None): + netloc = url.netloc + if mirrors and netloc in mirrors: + mirror = mirrors[netloc] + return '%sv2%s' % (mirror, path) + else: + if insecure: + scheme = 'http' + else: + scheme = 'https' + if netloc == 'docker.io': + netloc = 'registry-1.docker.io' + return '%s://%s/v2%s' % (scheme, netloc, path) @classmethod @tenacity.retry( # Retry up to 5 times with jittered exponential backoff @@ -326,24 +330,21 @@ class BaseImageUploader(object): wait=tenacity.wait_random_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(5) ) - def _inspect(cls, image_url, insecure=False, session=None): - original_image_url = image_url - image_url = cls._fix_dockerio_url(image_url) - parts = { - 'netloc': image_url.netloc - } - if insecure: - parts['scheme'] = 'http' - else: - parts['scheme'] = 'https' + def _inspect(cls, image_url, insecure=False, session=None, mirrors=None): image, tag = image_url.path.split(':') - parts['image'] = image - parts['tag'] = tag + parts = { + 'image': image, + 'tag': tag + } - manifest_url = ('%(scheme)s://%(netloc)s/v2' - '%(image)s/manifests/%(tag)s' % parts) - tags_url = ('%(scheme)s://%(netloc)s/v2' - '%(image)s/tags/list' % parts) + manifest_url = cls._build_url( + image_url, '%(image)s/manifests/%(tag)s' % parts, + insecure, mirrors + ) + tags_url = cls._build_url( + image_url, '%(image)s/tags/list' % parts, + insecure, mirrors + ) manifest_headers = { 'Accept': 'application/vnd.docker.distribution.manifest.v2+json' } @@ -369,8 +370,9 @@ class BaseImageUploader(object): config_headers = { 'Accept': manifest['config']['mediaType'] } - config_url = ('%(scheme)s://%(netloc)s/v2' - '%(image)s/blobs/%(config_digest)s' % parts) + config_url = cls._build_url( + image_url, '%(image)s/blobs/%(config_digest)s' % parts, + insecure, mirrors) config_f = p.submit( session.get, config_url, headers=config_headers, timeout=30) config_r = config_f.result() @@ -379,7 +381,7 @@ class BaseImageUploader(object): tags = tags_r.json()['tags'] digest = manifest_r.headers['Docker-Content-Digest'] config = config_r.json() - name = '%s%s' % (original_image_url.netloc, image) + name = '%s%s' % (image_url.netloc, image) return { 'Name': name, @@ -445,7 +447,7 @@ class BaseImageUploader(object): ) return tag_label - def discover_image_tags(self, images, tag_from_label=None): + def discover_image_tags(self, images, tag_from_label=None, mirrors=None): image_urls = [self._image_to_url(i) for i in images] # prime self.insecure_registries by testing every image @@ -455,7 +457,7 @@ class BaseImageUploader(object): discover_args = [] for image in images: discover_args.append((image, tag_from_label, - self.insecure_registries)) + self.insecure_registries, mirrors)) p = futures.ThreadPoolExecutor(max_workers=16) versioned_images = {} @@ -465,25 +467,28 @@ class BaseImageUploader(object): return versioned_images def discover_image_tag(self, image, tag_from_label=None, - fallback_tag=None, username=None, password=None): + fallback_tag=None, username=None, password=None, + mirrors=None): image_url = self._image_to_url(image) insecure = self.is_insecure_registry(image_url.netloc) session = self.authenticate( - image_url, insecure=insecure, username=username, password=password) - i = self._inspect(image_url, insecure, session) + image_url, insecure=insecure, username=username, password=password, + mirrors=mirrors) + i = self._inspect(image_url, insecure, session, mirrors=mirrors) return self._discover_tag_from_inspect(i, image, tag_from_label, fallback_tag) def filter_images_with_labels(self, images, labels, - username=None, password=None): + username=None, password=None, mirrors=None): images_with_labels = [] for image in images: url = self._image_to_url(image) insecure = self.is_insecure_registry(url.netloc) session = self.authenticate( - url, insecure=insecure, username=username, password=password) + url, insecure=insecure, username=username, password=password, + mirrors=mirrors) image_labels = self._image_labels( - url, insecure=insecure, session=session) + url, insecure=insecure, session=session, mirrors=mirrors) if set(labels).issubset(set(image_labels)): images_with_labels.append(image) @@ -558,17 +563,20 @@ class DockerImageUploader(BaseImageUploader): if t.modify_role: target_session = self.authenticate( - t.target_image_url, insecure=target_insecure) + t.target_image_url, insecure=target_insecure, + mirrors=t.mirrors) if self._image_exists(t.target_image, - session=target_session): + session=target_session, + mirrors=t.mirrors): LOG.warning('Skipping upload for modified image %s' % t.target_image) return [] else: source_session = self.authenticate( - t.source_image_url, insecure=source_insecure) + t.source_image_url, insecure=source_insecure, + mirrors=t.mirrors) if self._images_match(t.source_image, t.target_image, - session1=source_session): + session1=source_session, mirrors=t.mirrors): LOG.warning('Skipping upload for image %s' % t.image_name) return [] @@ -689,21 +697,25 @@ class SkopeoImageUploader(BaseImageUploader): return [] target_session = self.authenticate( - t.target_image_url, insecure=target_insecure) + t.target_image_url, insecure=target_insecure, + mirrors=t.mirrors) if t.modify_role and self._image_exists( - t.target_image, target_session): + t.target_image, target_session, + mirrors=t.mirrors): LOG.warning('Skipping upload for modified image %s' % t.target_image) return [] source_session = self.authenticate( - t.source_image_url, insecure=source_insecure) + t.source_image_url, insecure=source_insecure, + mirrors=t.mirrors) source_inspect = self._inspect( t.source_image_url, insecure=source_insecure, - session=source_session) + session=source_session, + mirrors=t.mirrors) source_layers = source_inspect.get('Layers', []) self._cross_repo_mount( t.target_image_url, self.image_layers, source_layers, @@ -835,7 +847,8 @@ class SkopeoImageUploader(BaseImageUploader): class UploadTask(object): def __init__(self, image_name, pull_source, push_destination, - append_tag, modify_role, modify_vars, dry_run, cleanup): + append_tag, modify_role, modify_vars, dry_run, cleanup, + mirrors): self.image_name = image_name self.pull_source = pull_source self.push_destination = push_destination @@ -844,6 +857,7 @@ class UploadTask(object): self.modify_vars = modify_vars self.dry_run = dry_run self.cleanup = cleanup + self.mirrors = mirrors if ':' in image_name: image = image_name.rpartition(':')[0] @@ -875,12 +889,13 @@ def upload_task(args): def discover_tag_from_inspect(args): - image, tag_from_label, insecure_registries = args + image, tag_from_label, insecure_registries, mirrors = args image_url = BaseImageUploader._image_to_url(image) insecure = image_url.netloc in insecure_registries - session = BaseImageUploader.authenticate(image_url, insecure=insecure) + session = BaseImageUploader.authenticate(image_url, insecure=insecure, + mirrors=mirrors) i = BaseImageUploader._inspect(image_url, insecure=insecure, - session=session) + session=session, mirrors=mirrors) if ':' in image_url.path: # break out the tag from the url to be the fallback tag path = image.rpartition(':') diff --git a/tripleo_common/image/kolla_builder.py b/tripleo_common/image/kolla_builder.py index ee204254e..c1a5ecae8 100644 --- a/tripleo_common/image/kolla_builder.py +++ b/tripleo_common/image/kolla_builder.py @@ -141,6 +141,11 @@ def container_images_prepare_multi(environment, roles_data, dry_run=False, if not cip: return + mirrors = {} + mirror = pd.get('DockerRegistryMirror') + if mirror: + mirrors['docker.io'] = mirror + env_params = {} service_filter = build_service_filter(environment, roles_data) @@ -176,6 +181,7 @@ def container_images_prepare_multi(environment, roles_data, dry_run=False, modify_role=modify_role, modify_vars=modify_vars, modify_only_with_labels=modify_only_with_labels, + mirrors=mirrors ) env_params.update(prepare_data['image_params']) @@ -187,7 +193,8 @@ def container_images_prepare_multi(environment, roles_data, dry_run=False, uploader = image_uploader.ImageUploadManager( [f.name], dry_run=dry_run, - cleanup=cleanup + cleanup=cleanup, + mirrors=mirrors ) uploader.upload() return env_params @@ -209,7 +216,8 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE, mapping_args=None, output_env_file=None, output_images_file=None, tag_from_label=None, append_tag=None, modify_role=None, - modify_vars=None, modify_only_with_labels=None): + modify_vars=None, modify_only_with_labels=None, + mirrors=None): """Perform container image preparation :param template_file: path to Jinja2 file containing all image entries @@ -235,6 +243,7 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE, :param modify_vars: dict of variables to pass to modify_role :param modify_only_with_labels: only modify the container images with the given labels + :param mirrors: dict of registry netloc values to mirror urls :returns: dict with entries for the supplied output_env_file or output_images_file """ @@ -269,7 +278,7 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE, if tag_from_label: image_version_tags = uploader.discover_image_tags( - images, tag_from_label) + images, tag_from_label, mirrors=mirrors) for entry in result: imagename = entry.get('imagename', '') image_no_tag = imagename.rpartition(':')[0] @@ -279,7 +288,7 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE, if modify_only_with_labels: images_with_labels = uploader.filter_images_with_labels( - images, modify_only_with_labels) + images, modify_only_with_labels, mirrors=mirrors) params = {} modify_append_tag = append_tag diff --git a/tripleo_common/tests/image/test_image_uploader.py b/tripleo_common/tests/image/test_image_uploader.py index 78e46f267..f30f8bc58 100644 --- a/tripleo_common/tests/image/test_image_uploader.py +++ b/tripleo_common/tests/image/test_image_uploader.py @@ -285,63 +285,63 @@ class TestBaseImageUploader(base.TestCase): self.assertEqual( ('docker.io/t/foo', 'a'), image_uploader.discover_tag_from_inspect( - ('docker.io/t/foo', 'rdo_version', sr)) + ('docker.io/t/foo', 'rdo_version', sr, None)) ) # templated labels -> tag self.assertEqual( ('docker.io/t/foo', '1.0.0-20180125'), image_uploader.discover_tag_from_inspect( - ('docker.io/t/foo', '{release}-{version}', sr)) + ('docker.io/t/foo', '{release}-{version}', sr, None)) ) # simple label -> tag with fallback self.assertEqual( ('docker.io/t/foo', 'a'), image_uploader.discover_tag_from_inspect( - ('docker.io/t/foo:a', 'bar', sr)) + ('docker.io/t/foo:a', 'bar', sr, None)) ) # templated labels -> tag with fallback self.assertEqual( ('docker.io/t/foo', 'a'), image_uploader.discover_tag_from_inspect( - ('docker.io/t/foo:a', '{releases}-{versions}', sr)) + ('docker.io/t/foo:a', '{releases}-{versions}', sr, None)) ) # Invalid template self.assertRaises( ImageUploaderException, image_uploader.discover_tag_from_inspect, - ('docker.io/t/foo', '{release}-{version', sr) + ('docker.io/t/foo', '{release}-{version', sr, None) ) # Missing label in template self.assertRaises( ImageUploaderException, image_uploader.discover_tag_from_inspect, - ('docker.io/t/foo', '{releases}-{version}', sr) + ('docker.io/t/foo', '{releases}-{version}', sr, None) ) # no tag_from_label specified self.assertRaises( ImageUploaderException, image_uploader.discover_tag_from_inspect, - ('docker.io/t/foo', None, sr) + ('docker.io/t/foo', None, sr, None) ) # missing RepoTags entry self.assertRaises( ImageUploaderException, image_uploader.discover_tag_from_inspect, - ('docker.io/t/foo', 'build_version', sr) + ('docker.io/t/foo', 'build_version', sr, None) ) # missing Labels entry self.assertRaises( ImageUploaderException, image_uploader.discover_tag_from_inspect, - ('docker.io/t/foo', 'version', sr) + ('docker.io/t/foo', 'version', sr, None) ) # inspect call failed @@ -349,7 +349,7 @@ class TestBaseImageUploader(base.TestCase): self.assertRaises( ImageUploaderException, image_uploader.discover_tag_from_inspect, - ('docker.io/t/foo', 'rdo_version', sr) + ('docker.io/t/foo', 'rdo_version', sr, None) ) @mock.patch('concurrent.futures.ThreadPoolExecutor') @@ -375,9 +375,9 @@ class TestBaseImageUploader(base.TestCase): mock_pool.return_value.map.assert_called_once_with( image_uploader.discover_tag_from_inspect, [ - ('docker.io/t/foo', 'rdo_release', set()), - ('docker.io/t/bar', 'rdo_release', set()), - ('docker.io/t/baz', 'rdo_release', set()) + ('docker.io/t/foo', 'rdo_release', set(), None), + ('docker.io/t/bar', 'rdo_release', set(), None), + ('docker.io/t/baz', 'rdo_release', set(), None) ]) @mock.patch('tripleo_common.image.image_uploader.' @@ -450,17 +450,38 @@ class TestBaseImageUploader(base.TestCase): auth(url1).headers['Authorization'] ) - def test_fix_dockerio_url(self): + def test_build_url(self): url1 = urlparse('docker://docker.io/t/nova-api:latest') url2 = urlparse('docker://registry-1.docker.io/t/nova-api:latest') url3 = urlparse('docker://192.0.2.1:8787/t/nova-api:latest') - fix = image_uploader.BaseImageUploader._fix_dockerio_url + build = image_uploader.BaseImageUploader._build_url # fix urls - self.assertEqual(url2, fix(url1)) + self.assertEqual( + 'https://registry-1.docker.io/v2/', + build(url1, '/') + ) # no change urls - self.assertEqual(url2, fix(url2)) - self.assertEqual(url3, fix(url3)) + self.assertEqual( + 'http://registry-1.docker.io/v2/t/nova-api/manifests/latest', + build(url2, '/t/nova-api/manifests/latest', + insecure=True) + ) + self.assertEqual( + 'https://192.0.2.1:8787/v2/t/nova-api/tags/list', + build(url3, '/t/nova-api/tags/list') + ) + + # test mirrors + mirrors = { + 'docker.io': 'http://192.0.2.2:8081/registry-1.docker/' + } + self.assertEqual( + 'http://192.0.2.2:8081/registry-1.docker/v2/' + 't/nova-api/blobs/asdf1234', + build(url1, '/t/nova-api/blobs/asdf1234', + mirrors=mirrors) + ) def test_inspect(self): req = self.requests @@ -577,7 +598,8 @@ class TestDockerImageUploader(base.TestCase): None, None, False, - 'full') + 'full', + None) ) ) @@ -616,7 +638,8 @@ class TestDockerImageUploader(base.TestCase): None, None, False, - 'full') + 'full', + None) ) ) @@ -669,7 +692,8 @@ class TestDockerImageUploader(base.TestCase): 'add-foo-plugin', {'foo_version': '1.0.1'}, False, - 'partial') + 'partial', + None) ) ) @@ -712,7 +736,8 @@ class TestDockerImageUploader(base.TestCase): self.uploader.upload_image, image_uploader.UploadTask( image + ':' + tag, None, push_destination, append_tag, 'add-foo-plugin', {'foo_version': '1.0.1'}, - False, 'full') + False, 'full', + None) ) self.dockermock.assert_called_once_with( @@ -743,7 +768,8 @@ class TestDockerImageUploader(base.TestCase): 'add-foo-plugin', {'foo_version': '1.0.1'}, True, - 'full') + 'full', + None) ) self.dockermock.assert_not_called() @@ -774,7 +800,8 @@ class TestDockerImageUploader(base.TestCase): 'add-foo-plugin', {'foo_version': '1.0.1'}, False, - 'full') + 'full', + None) ) self.dockermock.assert_not_called() @@ -894,7 +921,8 @@ class TestSkopeoImageUploader(base.TestCase): None, None, False, - 'full') + 'full', + None) ) ) mock_popen.assert_called_once_with([ @@ -958,14 +986,15 @@ class TestSkopeoImageUploader(base.TestCase): 'add-foo-plugin', {'foo_version': '1.0.1'}, False, - 'partial') + 'partial', + None) ) ) mock_inspect.assert_has_calls([ mock.call(urlparse( 'docker://docker.io/t/nova-api:latest' - ), insecure=False, session=mock.ANY) + ), insecure=False, mirrors=None, session=mock.ANY) ]) mock_copy.assert_has_calls([ mock.call( @@ -1014,7 +1043,7 @@ class TestSkopeoImageUploader(base.TestCase): self.uploader.upload_image, image_uploader.UploadTask( image + ':' + tag, None, push_destination, append_tag, 'add-foo-plugin', {'foo_version': '1.0.1'}, - False, 'full') + False, 'full', None) ) mock_copy.assert_called_once_with( @@ -1042,7 +1071,8 @@ class TestSkopeoImageUploader(base.TestCase): 'add-foo-plugin', {'foo_version': '1.0.1'}, True, - 'full') + 'full', + None) ) mock_ansible.assert_not_called() @@ -1072,7 +1102,8 @@ class TestSkopeoImageUploader(base.TestCase): 'add-foo-plugin', {'foo_version': '1.0.1'}, False, - 'full') + 'full', + None) ) mock_ansible.assert_not_called() diff --git a/tripleo_common/tests/image/test_kolla_builder.py b/tripleo_common/tests/image/test_kolla_builder.py index e0f083611..734f720dd 100644 --- a/tripleo_common/tests/image/test_kolla_builder.py +++ b/tripleo_common/tests/image/test_kolla_builder.py @@ -840,6 +840,7 @@ class TestPrepare(base.TestCase): env = { 'parameter_defaults': { 'LocalContainerRegistry': '192.0.2.1', + 'DockerRegistryMirror': 'http://192.0.2.2/reg/', 'ContainerImagePrepare': [{ 'set': mapping_args, 'tag_from_label': 'foo', @@ -896,7 +897,10 @@ class TestPrepare(base.TestCase): append_tag=mock.ANY, modify_role=None, modify_only_with_labels=None, - modify_vars=None + modify_vars=None, + mirrors={ + 'docker.io': 'http://192.0.2.2/reg/' + } ), mock.call( excludes=['nova', 'neutron'], @@ -911,7 +915,10 @@ class TestPrepare(base.TestCase): append_tag=mock.ANY, modify_role='add-foo-plugin', modify_only_with_labels=['kolla_version'], - modify_vars={'foo_version': '1.0.1'} + modify_vars={'foo_version': '1.0.1'}, + mirrors={ + 'docker.io': 'http://192.0.2.2/reg/' + } ) ]) @@ -995,7 +1002,8 @@ class TestPrepare(base.TestCase): append_tag=mock.ANY, modify_role=None, modify_only_with_labels=None, - modify_vars=None + modify_vars=None, + mirrors={} ), mock.call( excludes=['nova', 'neutron'], @@ -1010,11 +1018,13 @@ class TestPrepare(base.TestCase): append_tag=mock.ANY, modify_role='add-foo-plugin', modify_only_with_labels=['kolla_version'], - modify_vars={'foo_version': '1.0.1'} + modify_vars={'foo_version': '1.0.1'}, + mirrors={} ) ]) - mock_im.assert_called_once_with(mock.ANY, dry_run=True, cleanup='full') + mock_im.assert_called_once_with(mock.ANY, dry_run=True, cleanup='full', + mirrors={}) self.assertEqual( {