openstack
/
tripleo-common
Code Issues Proposed changes

Use DockerRegistryMirror to proxy requests 

This change consumes the DockerRegistryMirror parameter and uses that to proxy
requests to docker.io for auth and inspect calls.

This should improve the reliability of CI when unproxied calls to
docker.io can fail.

Change-Id: I11664091272d9bd216788217c0293b31d88e2758
Depends-On: If896c22bf449a3ac91ca363648f84dd5b9aef227
Closes-Bug: #1800958
This commit is contained in:
Steve Baker 2018-11-01 14:42:44 +13:00
parent 0c0a25ac93
commit e703ec1aa8
4 changed files with 171 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

149
tripleo_common/image/image_uploader.py
View File

@ -77,7 +77,8 @@ class ImageUploadManager(BaseImageManager):
"""
def __init__(self, config_files=None,
dry_run=False, cleanup=CLEANUP_FULL):
dry_run=False, cleanup=CLEANUP_FULL,
mirrors=None):
if config_files is None:
config_files = []
super(ImageUploadManager, self).__init__(config_files)
@ -87,13 +88,14 @@ class ImageUploadManager(BaseImageManager):
}
self.dry_run = dry_run
self.cleanup = cleanup
self.mirrors = mirrors
def discover_image_tag(self, image, tag_from_label=None,
username=None, password=None):
uploader = self.uploader(DEFAULT_UPLOADER)
return uploader.discover_image_tag(
image, tag_from_label=tag_from_label,
username=username, password=password)
username=username, password=password, mirrors=self.mirrors)
def uploader(self, uploader):
if uploader not in self.uploaders:
@ -136,7 +138,7 @@ class ImageUploadManager(BaseImageManager):
task = UploadTask(
image_name, pull_source, push_destination,
append_tag, modify_role, modify_vars, self.dry_run,
self.cleanup)
self.cleanup, self.mirrors)
uploader.add_upload_task(task)
for uploader in self.uploaders.values():
@ -221,13 +223,14 @@ class BaseImageUploader(object):
'Modifying image %s failed' % target_image)
@classmethod
def _images_match(cls, image1, image2, session1=None):
def _images_match(cls, image1, image2, session1=None, mirrors=None):
try:
image1_digest = cls._image_digest(image1, session=session1)
image1_digest = cls._image_digest(image1, session=session1,
mirrors=mirrors)
except Exception:
return False
try:
image2_digest = cls._image_digest(image2)
image2_digest = cls._image_digest(image2, mirrors=mirrors)
except Exception:
return False
@ -237,22 +240,22 @@ class BaseImageUploader(object):
return image1_digest == image2_digest
@classmethod
def _image_digest(cls, image, session=None):
def _image_digest(cls, image, session=None, mirrors=None):
image_url = cls._image_to_url(image)
insecure = image_url.netloc in cls.insecure_registries
i = cls._inspect(image_url, insecure, session)
i = cls._inspect(image_url, insecure, session, mirrors=mirrors)
return i.get('Digest')
@classmethod
def _image_labels(cls, image_url, insecure, session=None):
i = cls._inspect(image_url, insecure, session)
def _image_labels(cls, image_url, insecure, session=None, mirrors=None):
i = cls._inspect(image_url, insecure, session, mirrors=mirrors)
return i.get('Labels', {}) or {}
@classmethod
def _image_exists(cls, image, session=None):
def _image_exists(cls, image, session=None, mirrors=None):
try:
cls._image_digest(
image, session=session)
image, session=session, mirrors=mirrors)
except ImageNotFoundException:
return False
else:
@ -268,15 +271,11 @@ class BaseImageUploader(object):
stop=tenacity.stop_after_attempt(5)
)
def authenticate(cls, image_url, username=None, password=None,
insecure=False):
image_url = cls._fix_dockerio_url(image_url)
netloc = image_url.netloc
if insecure:
scheme = 'http'
else:
scheme = 'https'
insecure=False, mirrors=None):
image, tag = image_url.path.split(':')
url = '%s://%s/v2/' % (scheme, netloc)
url = cls._build_url(image_url, path='/',
insecure=insecure,
mirrors=mirrors)
session = requests.Session()
r = session.get(url, timeout=30)
LOG.debug('%s status code %s' % (url, r.status_code))
@ -308,14 +307,19 @@ class BaseImageUploader(object):
return session
@classmethod
def _fix_dockerio_url(cls, url):
one = 'docker.io'
two = 'registry-1.docker.io'
if url.netloc != one:
return url
return parse.ParseResult(url.scheme, two,
url.path, url.params,
url.query, url.fragment)
def _build_url(cls, url, path, insecure=False, mirrors=None):
netloc = url.netloc
if mirrors and netloc in mirrors:
mirror = mirrors[netloc]
return '%sv2%s' % (mirror, path)
else:
if insecure:
scheme = 'http'
else:
scheme = 'https'
if netloc == 'docker.io':
netloc = 'registry-1.docker.io'
return '%s://%s/v2%s' % (scheme, netloc, path)
@classmethod
@tenacity.retry( # Retry up to 5 times with jittered exponential backoff
@ -326,24 +330,21 @@ class BaseImageUploader(object):
wait=tenacity.wait_random_exponential(multiplier=1, max=10),
stop=tenacity.stop_after_attempt(5)
)
def _inspect(cls, image_url, insecure=False, session=None):
original_image_url = image_url
image_url = cls._fix_dockerio_url(image_url)
parts = {
'netloc': image_url.netloc
}
if insecure:
parts['scheme'] = 'http'
else:
parts['scheme'] = 'https'
def _inspect(cls, image_url, insecure=False, session=None, mirrors=None):
image, tag = image_url.path.split(':')
parts['image'] = image
parts['tag'] = tag
parts = {
'image': image,
'tag': tag
}
manifest_url = ('%(scheme)s://%(netloc)s/v2'
'%(image)s/manifests/%(tag)s' % parts)
tags_url = ('%(scheme)s://%(netloc)s/v2'
'%(image)s/tags/list' % parts)
manifest_url = cls._build_url(
image_url, '%(image)s/manifests/%(tag)s' % parts,
insecure, mirrors
)
tags_url = cls._build_url(
image_url, '%(image)s/tags/list' % parts,
insecure, mirrors
)
manifest_headers = {
'Accept': 'application/vnd.docker.distribution.manifest.v2+json'
}
@ -369,8 +370,9 @@ class BaseImageUploader(object):
config_headers = {
'Accept': manifest['config']['mediaType']
}
config_url = ('%(scheme)s://%(netloc)s/v2'
'%(image)s/blobs/%(config_digest)s' % parts)
config_url = cls._build_url(
image_url, '%(image)s/blobs/%(config_digest)s' % parts,
insecure, mirrors)
config_f = p.submit(
session.get, config_url, headers=config_headers, timeout=30)
config_r = config_f.result()
@ -379,7 +381,7 @@ class BaseImageUploader(object):
tags = tags_r.json()['tags']
digest = manifest_r.headers['Docker-Content-Digest']
config = config_r.json()
name = '%s%s' % (original_image_url.netloc, image)
name = '%s%s' % (image_url.netloc, image)
return {
'Name': name,
@ -445,7 +447,7 @@ class BaseImageUploader(object):
)
return tag_label
def discover_image_tags(self, images, tag_from_label=None):
def discover_image_tags(self, images, tag_from_label=None, mirrors=None):
image_urls = [self._image_to_url(i) for i in images]
# prime self.insecure_registries by testing every image
@ -455,7 +457,7 @@ class BaseImageUploader(object):
discover_args = []
for image in images:
discover_args.append((image, tag_from_label,
self.insecure_registries))
self.insecure_registries, mirrors))
p = futures.ThreadPoolExecutor(max_workers=16)
versioned_images = {}
@ -465,25 +467,28 @@ class BaseImageUploader(object):
return versioned_images
def discover_image_tag(self, image, tag_from_label=None,
fallback_tag=None, username=None, password=None):
fallback_tag=None, username=None, password=None,
mirrors=None):
image_url = self._image_to_url(image)
insecure = self.is_insecure_registry(image_url.netloc)
session = self.authenticate(
image_url, insecure=insecure, username=username, password=password)
i = self._inspect(image_url, insecure, session)
image_url, insecure=insecure, username=username, password=password,
mirrors=mirrors)
i = self._inspect(image_url, insecure, session, mirrors=mirrors)
return self._discover_tag_from_inspect(i, image, tag_from_label,
fallback_tag)
def filter_images_with_labels(self, images, labels,
username=None, password=None):
username=None, password=None, mirrors=None):
images_with_labels = []
for image in images:
url = self._image_to_url(image)
insecure = self.is_insecure_registry(url.netloc)
session = self.authenticate(
url, insecure=insecure, username=username, password=password)
url, insecure=insecure, username=username, password=password,
mirrors=mirrors)
image_labels = self._image_labels(
url, insecure=insecure, session=session)
url, insecure=insecure, session=session, mirrors=mirrors)
if set(labels).issubset(set(image_labels)):
images_with_labels.append(image)
@ -558,17 +563,20 @@ class DockerImageUploader(BaseImageUploader):
if t.modify_role:
target_session = self.authenticate(
t.target_image_url, insecure=target_insecure)
t.target_image_url, insecure=target_insecure,
mirrors=t.mirrors)
if self._image_exists(t.target_image,
session=target_session):
session=target_session,
mirrors=t.mirrors):
LOG.warning('Skipping upload for modified image %s' %
t.target_image)
return []
else:
source_session = self.authenticate(
t.source_image_url, insecure=source_insecure)
t.source_image_url, insecure=source_insecure,
mirrors=t.mirrors)
if self._images_match(t.source_image, t.target_image,
session1=source_session):
session1=source_session, mirrors=t.mirrors):
LOG.warning('Skipping upload for image %s' % t.image_name)
return []
@ -689,21 +697,25 @@ class SkopeoImageUploader(BaseImageUploader):
return []
target_session = self.authenticate(
t.target_image_url, insecure=target_insecure)
t.target_image_url, insecure=target_insecure,
mirrors=t.mirrors)
if t.modify_role and self._image_exists(
t.target_image, target_session):
t.target_image, target_session,
mirrors=t.mirrors):
LOG.warning('Skipping upload for modified image %s' %
t.target_image)
return []
source_session = self.authenticate(
t.source_image_url, insecure=source_insecure)
t.source_image_url, insecure=source_insecure,
mirrors=t.mirrors)
source_inspect = self._inspect(
t.source_image_url,
insecure=source_insecure,
session=source_session)
session=source_session,
mirrors=t.mirrors)
source_layers = source_inspect.get('Layers', [])
self._cross_repo_mount(
t.target_image_url, self.image_layers, source_layers,
@ -835,7 +847,8 @@ class SkopeoImageUploader(BaseImageUploader):
class UploadTask(object):
def __init__(self, image_name, pull_source, push_destination,
append_tag, modify_role, modify_vars, dry_run, cleanup):
append_tag, modify_role, modify_vars, dry_run, cleanup,
mirrors):
self.image_name = image_name
self.pull_source = pull_source
self.push_destination = push_destination
@ -844,6 +857,7 @@ class UploadTask(object):
self.modify_vars = modify_vars
self.dry_run = dry_run
self.cleanup = cleanup
self.mirrors = mirrors
if ':' in image_name:
image = image_name.rpartition(':')[0]
@ -875,12 +889,13 @@ def upload_task(args):
def discover_tag_from_inspect(args):
image, tag_from_label, insecure_registries = args
image, tag_from_label, insecure_registries, mirrors = args
image_url = BaseImageUploader._image_to_url(image)
insecure = image_url.netloc in insecure_registries
session = BaseImageUploader.authenticate(image_url, insecure=insecure)
session = BaseImageUploader.authenticate(image_url, insecure=insecure,
mirrors=mirrors)
i = BaseImageUploader._inspect(image_url, insecure=insecure,
session=session)
session=session, mirrors=mirrors)
if ':' in image_url.path:
# break out the tag from the url to be the fallback tag
path = image.rpartition(':')

17
tripleo_common/image/kolla_builder.py
View File

@ -141,6 +141,11 @@ def container_images_prepare_multi(environment, roles_data, dry_run=False,
if not cip:
return
mirrors = {}
mirror = pd.get('DockerRegistryMirror')
if mirror:
mirrors['docker.io'] = mirror
env_params = {}
service_filter = build_service_filter(environment, roles_data)
@ -176,6 +181,7 @@ def container_images_prepare_multi(environment, roles_data, dry_run=False,
modify_role=modify_role,
modify_vars=modify_vars,
modify_only_with_labels=modify_only_with_labels,
mirrors=mirrors
)
env_params.update(prepare_data['image_params'])
@ -187,7 +193,8 @@ def container_images_prepare_multi(environment, roles_data, dry_run=False,
uploader = image_uploader.ImageUploadManager(
[f.name],
dry_run=dry_run,
cleanup=cleanup
cleanup=cleanup,
mirrors=mirrors
)
uploader.upload()
return env_params
@ -209,7 +216,8 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
mapping_args=None, output_env_file=None,
output_images_file=None, tag_from_label=None,
append_tag=None, modify_role=None,
modify_vars=None, modify_only_with_labels=None):
modify_vars=None, modify_only_with_labels=None,
mirrors=None):
"""Perform container image preparation
:param template_file: path to Jinja2 file containing all image entries
@ -235,6 +243,7 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
:param modify_vars: dict of variables to pass to modify_role
:param modify_only_with_labels: only modify the container images with the
given labels
:param mirrors: dict of registry netloc values to mirror urls
:returns: dict with entries for the supplied output_env_file or
output_images_file
"""
@ -269,7 +278,7 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
if tag_from_label:
image_version_tags = uploader.discover_image_tags(
images, tag_from_label)
images, tag_from_label, mirrors=mirrors)
for entry in result:
imagename = entry.get('imagename', '')
image_no_tag = imagename.rpartition(':')[0]
@ -279,7 +288,7 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
if modify_only_with_labels:
images_with_labels = uploader.filter_images_with_labels(
images, modify_only_with_labels)
images, modify_only_with_labels, mirrors=mirrors)
params = {}
modify_append_tag = append_tag

91
tripleo_common/tests/image/test_image_uploader.py
View File

@ -285,63 +285,63 @@ class TestBaseImageUploader(base.TestCase):
self.assertEqual(
('docker.io/t/foo', 'a'),
image_uploader.discover_tag_from_inspect(
('docker.io/t/foo', 'rdo_version', sr))
('docker.io/t/foo', 'rdo_version', sr, None))
)
# templated labels -> tag
self.assertEqual(
('docker.io/t/foo', '1.0.0-20180125'),
image_uploader.discover_tag_from_inspect(
('docker.io/t/foo', '{release}-{version}', sr))
('docker.io/t/foo', '{release}-{version}', sr, None))
)
# simple label -> tag with fallback
self.assertEqual(
('docker.io/t/foo', 'a'),
image_uploader.discover_tag_from_inspect(
('docker.io/t/foo:a', 'bar', sr))
('docker.io/t/foo:a', 'bar', sr, None))
)
# templated labels -> tag with fallback
self.assertEqual(
('docker.io/t/foo', 'a'),
image_uploader.discover_tag_from_inspect(
('docker.io/t/foo:a', '{releases}-{versions}', sr))
('docker.io/t/foo:a', '{releases}-{versions}', sr, None))
)
# Invalid template
self.assertRaises(
ImageUploaderException,
image_uploader.discover_tag_from_inspect,
('docker.io/t/foo', '{release}-{version', sr)
('docker.io/t/foo', '{release}-{version', sr, None)
)
# Missing label in template
self.assertRaises(
ImageUploaderException,
image_uploader.discover_tag_from_inspect,
('docker.io/t/foo', '{releases}-{version}', sr)
('docker.io/t/foo', '{releases}-{version}', sr, None)
)
# no tag_from_label specified
self.assertRaises(
ImageUploaderException,
image_uploader.discover_tag_from_inspect,
('docker.io/t/foo', None, sr)
('docker.io/t/foo', None, sr, None)
)
# missing RepoTags entry
self.assertRaises(
ImageUploaderException,
image_uploader.discover_tag_from_inspect,
('docker.io/t/foo', 'build_version', sr)
('docker.io/t/foo', 'build_version', sr, None)
)
# missing Labels entry
self.assertRaises(
ImageUploaderException,
image_uploader.discover_tag_from_inspect,
('docker.io/t/foo', 'version', sr)
('docker.io/t/foo', 'version', sr, None)
)
# inspect call failed
@ -349,7 +349,7 @@ class TestBaseImageUploader(base.TestCase):
self.assertRaises(
ImageUploaderException,
image_uploader.discover_tag_from_inspect,
('docker.io/t/foo', 'rdo_version', sr)
('docker.io/t/foo', 'rdo_version', sr, None)
)
@mock.patch('concurrent.futures.ThreadPoolExecutor')
@ -375,9 +375,9 @@ class TestBaseImageUploader(base.TestCase):
mock_pool.return_value.map.assert_called_once_with(
image_uploader.discover_tag_from_inspect,
[
('docker.io/t/foo', 'rdo_release', set()),
('docker.io/t/bar', 'rdo_release', set()),
('docker.io/t/baz', 'rdo_release', set())
('docker.io/t/foo', 'rdo_release', set(), None),
('docker.io/t/bar', 'rdo_release', set(), None),
('docker.io/t/baz', 'rdo_release', set(), None)
])
@mock.patch('tripleo_common.image.image_uploader.'
@ -450,17 +450,38 @@ class TestBaseImageUploader(base.TestCase):
auth(url1).headers['Authorization']
)
def test_fix_dockerio_url(self):
def test_build_url(self):
url1 = urlparse('docker://docker.io/t/nova-api:latest')
url2 = urlparse('docker://registry-1.docker.io/t/nova-api:latest')
url3 = urlparse('docker://192.0.2.1:8787/t/nova-api:latest')
fix = image_uploader.BaseImageUploader._fix_dockerio_url
build = image_uploader.BaseImageUploader._build_url
# fix urls
self.assertEqual(url2, fix(url1))
self.assertEqual(
'https://registry-1.docker.io/v2/',
build(url1, '/')
)
# no change urls
self.assertEqual(url2, fix(url2))
self.assertEqual(url3, fix(url3))
self.assertEqual(
'http://registry-1.docker.io/v2/t/nova-api/manifests/latest',
build(url2, '/t/nova-api/manifests/latest',
insecure=True)
)
self.assertEqual(
'https://192.0.2.1:8787/v2/t/nova-api/tags/list',
build(url3, '/t/nova-api/tags/list')
)
# test mirrors
mirrors = {
'docker.io': 'http://192.0.2.2:8081/registry-1.docker/'
}
self.assertEqual(
'http://192.0.2.2:8081/registry-1.docker/v2/'
't/nova-api/blobs/asdf1234',
build(url1, '/t/nova-api/blobs/asdf1234',
mirrors=mirrors)
)
def test_inspect(self):
req = self.requests
@ -577,7 +598,8 @@ class TestDockerImageUploader(base.TestCase):
None,
None,
False,
'full')
'full',
None)
)
)
@ -616,7 +638,8 @@ class TestDockerImageUploader(base.TestCase):
None,
None,
False,
'full')
'full',
None)
)
)
@ -669,7 +692,8 @@ class TestDockerImageUploader(base.TestCase):
'add-foo-plugin',
{'foo_version': '1.0.1'},
False,
'partial')
'partial',
None)
)
)
@ -712,7 +736,8 @@ class TestDockerImageUploader(base.TestCase):
self.uploader.upload_image, image_uploader.UploadTask(
image + ':' + tag, None, push_destination,
append_tag, 'add-foo-plugin', {'foo_version': '1.0.1'},
False, 'full')
False, 'full',
None)
)
self.dockermock.assert_called_once_with(
@ -743,7 +768,8 @@ class TestDockerImageUploader(base.TestCase):
'add-foo-plugin',
{'foo_version': '1.0.1'},
True,
'full')
'full',
None)
)
self.dockermock.assert_not_called()
@ -774,7 +800,8 @@ class TestDockerImageUploader(base.TestCase):
'add-foo-plugin',
{'foo_version': '1.0.1'},
False,
'full')
'full',
None)
)
self.dockermock.assert_not_called()
@ -894,7 +921,8 @@ class TestSkopeoImageUploader(base.TestCase):
None,
None,
False,
'full')
'full',
None)
)
)
mock_popen.assert_called_once_with([
@ -958,14 +986,15 @@ class TestSkopeoImageUploader(base.TestCase):
'add-foo-plugin',
{'foo_version': '1.0.1'},
False,
'partial')
'partial',
None)
)
)
mock_inspect.assert_has_calls([
mock.call(urlparse(
'docker://docker.io/t/nova-api:latest'
), insecure=False, session=mock.ANY)
), insecure=False, mirrors=None, session=mock.ANY)
])
mock_copy.assert_has_calls([
mock.call(
@ -1014,7 +1043,7 @@ class TestSkopeoImageUploader(base.TestCase):
self.uploader.upload_image, image_uploader.UploadTask(
image + ':' + tag, None, push_destination,
append_tag, 'add-foo-plugin', {'foo_version': '1.0.1'},
False, 'full')
False, 'full', None)
)
mock_copy.assert_called_once_with(
@ -1042,7 +1071,8 @@ class TestSkopeoImageUploader(base.TestCase):
'add-foo-plugin',
{'foo_version': '1.0.1'},
True,
'full')
'full',
None)
)
mock_ansible.assert_not_called()
@ -1072,7 +1102,8 @@ class TestSkopeoImageUploader(base.TestCase):
'add-foo-plugin',
{'foo_version': '1.0.1'},
False,
'full')
'full',
None)
)
mock_ansible.assert_not_called()

20
tripleo_common/tests/image/test_kolla_builder.py
View File

@ -840,6 +840,7 @@ class TestPrepare(base.TestCase):
env = {
'parameter_defaults': {
'LocalContainerRegistry': '192.0.2.1',
'DockerRegistryMirror': 'http://192.0.2.2/reg/',
'ContainerImagePrepare': [{
'set': mapping_args,
'tag_from_label': 'foo',
@ -896,7 +897,10 @@ class TestPrepare(base.TestCase):
append_tag=mock.ANY,
modify_role=None,
modify_only_with_labels=None,
modify_vars=None
modify_vars=None,
mirrors={
'docker.io': 'http://192.0.2.2/reg/'
}
),
mock.call(
excludes=['nova', 'neutron'],
@ -911,7 +915,10 @@ class TestPrepare(base.TestCase):
append_tag=mock.ANY,
modify_role='add-foo-plugin',
modify_only_with_labels=['kolla_version'],
modify_vars={'foo_version': '1.0.1'}
modify_vars={'foo_version': '1.0.1'},
mirrors={
'docker.io': 'http://192.0.2.2/reg/'
}
)
])
@ -995,7 +1002,8 @@ class TestPrepare(base.TestCase):
append_tag=mock.ANY,
modify_role=None,
modify_only_with_labels=None,
modify_vars=None
modify_vars=None,
mirrors={}
),
mock.call(
excludes=['nova', 'neutron'],
@ -1010,11 +1018,13 @@ class TestPrepare(base.TestCase):
append_tag=mock.ANY,
modify_role='add-foo-plugin',
modify_only_with_labels=['kolla_version'],
modify_vars={'foo_version': '1.0.1'}
modify_vars={'foo_version': '1.0.1'},
mirrors={}
)
])
mock_im.assert_called_once_with(mock.ANY, dry_run=True, cleanup='full')
mock_im.assert_called_once_with(mock.ANY, dry_run=True, cleanup='full',
mirrors={})
self.assertEqual(
{