openstack/tripleo-common
Code Issues Proposed changes

Merge "Use DockerRegistryMirror to proxy requests"

Browse Source
This commit is contained in:
Zuul 2018-11-01 17:42:23 +00:00 committed by Gerrit Code Review
commit 1ee3c59dc7
4 changed files with 171 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

149
tripleo_common/image/image_uploader.py
View File

@ -77,7 +77,8 @@ class ImageUploadManager(BaseImageManager):
"""
def __init__(self, config_files=None,
dry_run=False, cleanup=CLEANUP_FULL):
dry_run=False, cleanup=CLEANUP_FULL,
mirrors=None):
if config_files is None:
config_files = []
super(ImageUploadManager, self).__init__(config_files)
@ -87,13 +88,14 @@ class ImageUploadManager(BaseImageManager):
}
self.dry_run = dry_run
self.cleanup = cleanup
self.mirrors = mirrors
def discover_image_tag(self, image, tag_from_label=None,
username=None, password=None):
uploader = self.uploader(DEFAULT_UPLOADER)
return uploader.discover_image_tag(
image, tag_from_label=tag_from_label,
username=username, password=password)
username=username, password=password, mirrors=self.mirrors)
def uploader(self, uploader):
if uploader not in self.uploaders:
@ -136,7 +138,7 @@ class ImageUploadManager(BaseImageManager):
task = UploadTask(
image_name, pull_source, push_destination,
append_tag, modify_role, modify_vars, self.dry_run,
self.cleanup)
self.cleanup, self.mirrors)
uploader.add_upload_task(task)
for uploader in self.uploaders.values():
@ -221,13 +223,14 @@ class BaseImageUploader(object):
'Modifying image %s failed' % target_image)
@classmethod
def _images_match(cls, image1, image2, session1=None):
def _images_match(cls, image1, image2, session1=None, mirrors=None):
try:
image1_digest = cls._image_digest(image1, session=session1)
image1_digest = cls._image_digest(image1, session=session1,
mirrors=mirrors)
except Exception:
return False
try:
image2_digest = cls._image_digest(image2)
image2_digest = cls._image_digest(image2, mirrors=mirrors)
except Exception:
return False
@ -237,22 +240,22 @@ class BaseImageUploader(object):
return image1_digest == image2_digest
@classmethod
def _image_digest(cls, image, session=None):
def _image_digest(cls, image, session=None, mirrors=None):
image_url = cls._image_to_url(image)
insecure = image_url.netloc in cls.insecure_registries
i = cls._inspect(image_url, insecure, session)
i = cls._inspect(image_url, insecure, session, mirrors=mirrors)
return i.get('Digest')
@classmethod
def _image_labels(cls, image_url, insecure, session=None):
i = cls._inspect(image_url, insecure, session)
def _image_labels(cls, image_url, insecure, session=None, mirrors=None):
i = cls._inspect(image_url, insecure, session, mirrors=mirrors)
return i.get('Labels', {}) or {}
@classmethod
def _image_exists(cls, image, session=None):
def _image_exists(cls, image, session=None, mirrors=None):
try:
cls._image_digest(
image, session=session)
image, session=session, mirrors=mirrors)
except ImageNotFoundException:
return False
else:
@ -268,15 +271,11 @@ class BaseImageUploader(object):
stop=tenacity.stop_after_attempt(5)
)
def authenticate(cls, image_url, username=None, password=None,
insecure=False):
image_url = cls._fix_dockerio_url(image_url)
netloc = image_url.netloc
if insecure:
scheme = 'http'
else:
scheme = 'https'
insecure=False, mirrors=None):
image, tag = image_url.path.split(':')
url = '%s://%s/v2/' % (scheme, netloc)
url = cls._build_url(image_url, path='/',
insecure=insecure,
mirrors=mirrors)
session = requests.Session()
r = session.get(url, timeout=30)
LOG.debug('%s status code %s' % (url, r.status_code))
@ -308,14 +307,19 @@ class BaseImageUploader(object):
return session
@classmethod
def _fix_dockerio_url(cls, url):
one = 'docker.io'
two = 'registry-1.docker.io'
if url.netloc != one:
return url
return parse.ParseResult(url.scheme, two,
url.path, url.params,
url.query, url.fragment)
def _build_url(cls, url, path, insecure=False, mirrors=None):
netloc = url.netloc
if mirrors and netloc in mirrors:
mirror = mirrors[netloc]
return '%sv2%s' % (mirror, path)
else:
if insecure:
scheme = 'http'
else:
scheme = 'https'
if netloc == 'docker.io':
netloc = 'registry-1.docker.io'
return '%s://%s/v2%s' % (scheme, netloc, path)
@classmethod
@tenacity.retry( # Retry up to 5 times with jittered exponential backoff
@ -326,24 +330,21 @@ class BaseImageUploader(object):
wait=tenacity.wait_random_exponential(multiplier=1, max=10),
stop=tenacity.stop_after_attempt(5)
)
def _inspect(cls, image_url, insecure=False, session=None):
original_image_url = image_url
image_url = cls._fix_dockerio_url(image_url)
parts = {
'netloc': image_url.netloc
}
if insecure:
parts['scheme'] = 'http'
else:
parts['scheme'] = 'https'
def _inspect(cls, image_url, insecure=False, session=None, mirrors=None):
image, tag = image_url.path.split(':')
parts['image'] = image
parts['tag'] = tag
parts = {
'image': image,
'tag': tag
}
manifest_url = ('%(scheme)s://%(netloc)s/v2'
'%(image)s/manifests/%(tag)s' % parts)
tags_url = ('%(scheme)s://%(netloc)s/v2'
'%(image)s/tags/list' % parts)
manifest_url = cls._build_url(
image_url, '%(image)s/manifests/%(tag)s' % parts,
insecure, mirrors
)
tags_url = cls._build_url(
image_url, '%(image)s/tags/list' % parts,
insecure, mirrors
)
manifest_headers = {
'Accept': 'application/vnd.docker.distribution.manifest.v2+json'
}
@ -369,8 +370,9 @@ class BaseImageUploader(object):
config_headers = {
'Accept': manifest['config']['mediaType']
}
config_url = ('%(scheme)s://%(netloc)s/v2'
'%(image)s/blobs/%(config_digest)s' % parts)
config_url = cls._build_url(
image_url, '%(image)s/blobs/%(config_digest)s' % parts,
insecure, mirrors)
config_f = p.submit(
session.get, config_url, headers=config_headers, timeout=30)
config_r = config_f.result()
@ -379,7 +381,7 @@ class BaseImageUploader(object):
tags = tags_r.json()['tags']
digest = manifest_r.headers['Docker-Content-Digest']
config = config_r.json()
name = '%s%s' % (original_image_url.netloc, image)
name = '%s%s' % (image_url.netloc, image)
return {
'Name': name,
@ -445,7 +447,7 @@ class BaseImageUploader(object):
)
return tag_label
def discover_image_tags(self, images, tag_from_label=None):
def discover_image_tags(self, images, tag_from_label=None, mirrors=None):
image_urls = [self._image_to_url(i) for i in images]
# prime self.insecure_registries by testing every image
@ -455,7 +457,7 @@ class BaseImageUploader(object):
discover_args = []
for image in images:
discover_args.append((image, tag_from_label,
self.insecure_registries))
self.insecure_registries, mirrors))
p = futures.ThreadPoolExecutor(max_workers=16)
versioned_images = {}
@ -465,25 +467,28 @@ class BaseImageUploader(object):
return versioned_images
def discover_image_tag(self, image, tag_from_label=None,
fallback_tag=None, username=None, password=None):
fallback_tag=None, username=None, password=None,
mirrors=None):
image_url = self._image_to_url(image)
insecure = self.is_insecure_registry(image_url.netloc)
session = self.authenticate(
image_url, insecure=insecure, username=username, password=password)
i = self._inspect(image_url, insecure, session)
image_url, insecure=insecure, username=username, password=password,
mirrors=mirrors)
i = self._inspect(image_url, insecure, session, mirrors=mirrors)
return self._discover_tag_from_inspect(i, image, tag_from_label,
fallback_tag)
def filter_images_with_labels(self, images, labels,
username=None, password=None):
username=None, password=None, mirrors=None):
images_with_labels = []
for image in images:
url = self._image_to_url(image)
insecure = self.is_insecure_registry(url.netloc)
session = self.authenticate(
url, insecure=insecure, username=username, password=password)
url, insecure=insecure, username=username, password=password,
mirrors=mirrors)
image_labels = self._image_labels(
url, insecure=insecure, session=session)
url, insecure=insecure, session=session, mirrors=mirrors)
if set(labels).issubset(set(image_labels)):
images_with_labels.append(image)
@ -558,17 +563,20 @@ class DockerImageUploader(BaseImageUploader):
if t.modify_role:
target_session = self.authenticate(
t.target_image_url, insecure=target_insecure)
t.target_image_url, insecure=target_insecure,
mirrors=t.mirrors)
if self._image_exists(t.target_image,
session=target_session):
session=target_session,
mirrors=t.mirrors):
LOG.warning('Skipping upload for modified image %s' %
t.target_image)
return []
else:
source_session = self.authenticate(
t.source_image_url, insecure=source_insecure)
t.source_image_url, insecure=source_insecure,
mirrors=t.mirrors)
if self._images_match(t.source_image, t.target_image,
session1=source_session):
session1=source_session, mirrors=t.mirrors):
LOG.warning('Skipping upload for image %s' % t.image_name)
return []
@ -689,21 +697,25 @@ class SkopeoImageUploader(BaseImageUploader):
return []
target_session = self.authenticate(
t.target_image_url, insecure=target_insecure)
t.target_image_url, insecure=target_insecure,
mirrors=t.mirrors)
if t.modify_role and self._image_exists(
t.target_image, target_session):
t.target_image, target_session,
mirrors=t.mirrors):
LOG.warning('Skipping upload for modified image %s' %
t.target_image)
return []
source_session = self.authenticate(
t.source_image_url, insecure=source_insecure)
t.source_image_url, insecure=source_insecure,
mirrors=t.mirrors)
source_inspect = self._inspect(
t.source_image_url,
insecure=source_insecure,
session=source_session)
session=source_session,
mirrors=t.mirrors)
source_layers = source_inspect.get('Layers', [])
self._cross_repo_mount(
t.target_image_url, self.image_layers, source_layers,
@ -835,7 +847,8 @@ class SkopeoImageUploader(BaseImageUploader):
class UploadTask(object):
def __init__(self, image_name, pull_source, push_destination,
append_tag, modify_role, modify_vars, dry_run, cleanup):
append_tag, modify_role, modify_vars, dry_run, cleanup,
mirrors):
self.image_name = image_name
self.pull_source = pull_source
self.push_destination = push_destination
@ -844,6 +857,7 @@ class UploadTask(object):
self.modify_vars = modify_vars
self.dry_run = dry_run
self.cleanup = cleanup
self.mirrors = mirrors
if ':' in image_name:
image = image_name.rpartition(':')[0]
@ -875,12 +889,13 @@ def upload_task(args):
def discover_tag_from_inspect(args):
image, tag_from_label, insecure_registries = args
image, tag_from_label, insecure_registries, mirrors = args
image_url = BaseImageUploader._image_to_url(image)
insecure = image_url.netloc in insecure_registries
session = BaseImageUploader.authenticate(image_url, insecure=insecure)
session = BaseImageUploader.authenticate(image_url, insecure=insecure,
mirrors=mirrors)
i = BaseImageUploader._inspect(image_url, insecure=insecure,
session=session)
session=session, mirrors=mirrors)
if ':' in image_url.path:
# break out the tag from the url to be the fallback tag
path = image.rpartition(':')

17
tripleo_common/image/kolla_builder.py
View File

@ -141,6 +141,11 @@ def container_images_prepare_multi(environment, roles_data, dry_run=False,
if not cip:
return
mirrors = {}
mirror = pd.get('DockerRegistryMirror')
if mirror:
mirrors['docker.io'] = mirror
env_params = {}
service_filter = build_service_filter(environment, roles_data)
@ -176,6 +181,7 @@ def container_images_prepare_multi(environment, roles_data, dry_run=False,
modify_role=modify_role,
modify_vars=modify_vars,
modify_only_with_labels=modify_only_with_labels,
mirrors=mirrors
)
env_params.update(prepare_data['image_params'])
@ -187,7 +193,8 @@ def container_images_prepare_multi(environment, roles_data, dry_run=False,
uploader = image_uploader.ImageUploadManager(
[f.name],
dry_run=dry_run,
cleanup=cleanup
cleanup=cleanup,
mirrors=mirrors
)
uploader.upload()
return env_params
@ -209,7 +216,8 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
mapping_args=None, output_env_file=None,
output_images_file=None, tag_from_label=None,
append_tag=None, modify_role=None,
modify_vars=None, modify_only_with_labels=None):
modify_vars=None, modify_only_with_labels=None,
mirrors=None):
"""Perform container image preparation
:param template_file: path to Jinja2 file containing all image entries
@ -235,6 +243,7 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
:param modify_vars: dict of variables to pass to modify_role
:param modify_only_with_labels: only modify the container images with the
given labels
:param mirrors: dict of registry netloc values to mirror urls
:returns: dict with entries for the supplied output_env_file or
output_images_file
"""
@ -269,7 +278,7 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
if tag_from_label:
image_version_tags = uploader.discover_image_tags(
images, tag_from_label)
images, tag_from_label, mirrors=mirrors)
for entry in result:
imagename = entry.get('imagename', '')
image_no_tag = imagename.rpartition(':')[0]
@ -279,7 +288,7 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
if modify_only_with_labels:
images_with_labels = uploader.filter_images_with_labels(
images, modify_only_with_labels)
images, modify_only_with_labels, mirrors=mirrors)
params = {}
modify_append_tag = append_tag

91
tripleo_common/tests/image/test_image_uploader.py
View File

@ -285,63 +285,63 @@ class TestBaseImageUploader(base.TestCase):
self.assertEqual(
('docker.io/t/foo', 'a'),
image_uploader.discover_tag_from_inspect(
('docker.io/t/foo', 'rdo_version', sr))
('docker.io/t/foo', 'rdo_version', sr, None))
)
# templated labels -> tag
self.assertEqual(
('docker.io/t/foo', '1.0.0-20180125'),
image_uploader.discover_tag_from_inspect(
('docker.io/t/foo', '{release}-{version}', sr))
('docker.io/t/foo', '{release}-{version}', sr, None))
)
# simple label -> tag with fallback
self.assertEqual(
('docker.io/t/foo', 'a'),
image_uploader.discover_tag_from_inspect(
('docker.io/t/foo:a', 'bar', sr))
('docker.io/t/foo:a', 'bar', sr, None))
)
# templated labels -> tag with fallback
self.assertEqual(
('docker.io/t/foo', 'a'),
image_uploader.discover_tag_from_inspect(
('docker.io/t/foo:a', '{releases}-{versions}', sr))
('docker.io/t/foo:a', '{releases}-{versions}', sr, None))
)
# Invalid template
self.assertRaises(
ImageUploaderException,
image_uploader.discover_tag_from_inspect,
('docker.io/t/foo', '{release}-{version', sr)
('docker.io/t/foo', '{release}-{version', sr, None)
)
# Missing label in template
self.assertRaises(
ImageUploaderException,
image_uploader.discover_tag_from_inspect,
('docker.io/t/foo', '{releases}-{version}', sr)
('docker.io/t/foo', '{releases}-{version}', sr, None)
)
# no tag_from_label specified
self.assertRaises(
ImageUploaderException,
image_uploader.discover_tag_from_inspect,
('docker.io/t/foo', None, sr)
('docker.io/t/foo', None, sr, None)
)
# missing RepoTags entry
self.assertRaises(
ImageUploaderException,
image_uploader.discover_tag_from_inspect,
('docker.io/t/foo', 'build_version', sr)
('docker.io/t/foo', 'build_version', sr, None)
)
# missing Labels entry
self.assertRaises(
ImageUploaderException,
image_uploader.discover_tag_from_inspect,
('docker.io/t/foo', 'version', sr)
('docker.io/t/foo', 'version', sr, None)
)
# inspect call failed
@ -349,7 +349,7 @@ class TestBaseImageUploader(base.TestCase):
self.assertRaises(
ImageUploaderException,
image_uploader.discover_tag_from_inspect,
('docker.io/t/foo', 'rdo_version', sr)
('docker.io/t/foo', 'rdo_version', sr, None)
)
@mock.patch('concurrent.futures.ThreadPoolExecutor')
@ -375,9 +375,9 @@ class TestBaseImageUploader(base.TestCase):
mock_pool.return_value.map.assert_called_once_with(
image_uploader.discover_tag_from_inspect,
[
('docker.io/t/foo', 'rdo_release', set()),
('docker.io/t/bar', 'rdo_release', set()),
('docker.io/t/baz', 'rdo_release', set())
('docker.io/t/foo', 'rdo_release', set(), None),
('docker.io/t/bar', 'rdo_release', set(), None),
('docker.io/t/baz', 'rdo_release', set(), None)
])
@mock.patch('tripleo_common.image.image_uploader.'
@ -450,17 +450,38 @@ class TestBaseImageUploader(base.TestCase):
auth(url1).headers['Authorization']
)
def test_fix_dockerio_url(self):
def test_build_url(self):
url1 = urlparse('docker://docker.io/t/nova-api:latest')
url2 = urlparse('docker://registry-1.docker.io/t/nova-api:latest')
url3 = urlparse('docker://192.0.2.1:8787/t/nova-api:latest')
fix = image_uploader.BaseImageUploader._fix_dockerio_url
build = image_uploader.BaseImageUploader._build_url
# fix urls
self.assertEqual(url2, fix(url1))
self.assertEqual(
'https://registry-1.docker.io/v2/',
build(url1, '/')
)
# no change urls
self.assertEqual(url2, fix(url2))
self.assertEqual(url3, fix(url3))
self.assertEqual(
'http://registry-1.docker.io/v2/t/nova-api/manifests/latest',
build(url2, '/t/nova-api/manifests/latest',
insecure=True)
)
self.assertEqual(
'https://192.0.2.1:8787/v2/t/nova-api/tags/list',
build(url3, '/t/nova-api/tags/list')
)
# test mirrors
mirrors = {
'docker.io': 'http://192.0.2.2:8081/registry-1.docker/'
}
self.assertEqual(
'http://192.0.2.2:8081/registry-1.docker/v2/'
't/nova-api/blobs/asdf1234',
build(url1, '/t/nova-api/blobs/asdf1234',
mirrors=mirrors)
)
def test_inspect(self):
req = self.requests
@ -577,7 +598,8 @@ class TestDockerImageUploader(base.TestCase):
None,
None,
False,
'full')
'full',
None)
)
)
@ -616,7 +638,8 @@ class TestDockerImageUploader(base.TestCase):
None,
None,
False,
'full')
'full',
None)
)
)
@ -669,7 +692,8 @@ class TestDockerImageUploader(base.TestCase):
'add-foo-plugin',
{'foo_version': '1.0.1'},
False,
'partial')
'partial',
None)
)
)
@ -712,7 +736,8 @@ class TestDockerImageUploader(base.TestCase):
self.uploader.upload_image, image_uploader.UploadTask(
image + ':' + tag, None, push_destination,
append_tag, 'add-foo-plugin', {'foo_version': '1.0.1'},
False, 'full')
False, 'full',
None)
)
self.dockermock.assert_called_once_with(
@ -743,7 +768,8 @@ class TestDockerImageUploader(base.TestCase):
'add-foo-plugin',
{'foo_version': '1.0.1'},
True,
'full')
'full',
None)
)
self.dockermock.assert_not_called()
@ -774,7 +800,8 @@ class TestDockerImageUploader(base.TestCase):
'add-foo-plugin',
{'foo_version': '1.0.1'},
False,
'full')
'full',
None)
)
self.dockermock.assert_not_called()
@ -894,7 +921,8 @@ class TestSkopeoImageUploader(base.TestCase):
None,
None,
False,
'full')
'full',
None)
)
)
mock_popen.assert_called_once_with([
@ -958,14 +986,15 @@ class TestSkopeoImageUploader(base.TestCase):
'add-foo-plugin',
{'foo_version': '1.0.1'},
False,
'partial')
'partial',
None)
)
)
mock_inspect.assert_has_calls([
mock.call(urlparse(
'docker://docker.io/t/nova-api:latest'
), insecure=False, session=mock.ANY)
), insecure=False, mirrors=None, session=mock.ANY)
])
mock_copy.assert_has_calls([
mock.call(
@ -1014,7 +1043,7 @@ class TestSkopeoImageUploader(base.TestCase):
self.uploader.upload_image, image_uploader.UploadTask(
image + ':' + tag, None, push_destination,
append_tag, 'add-foo-plugin', {'foo_version': '1.0.1'},
False, 'full')
False, 'full', None)
)
mock_copy.assert_called_once_with(
@ -1042,7 +1071,8 @@ class TestSkopeoImageUploader(base.TestCase):
'add-foo-plugin',
{'foo_version': '1.0.1'},
True,
'full')
'full',
None)
)
mock_ansible.assert_not_called()
@ -1072,7 +1102,8 @@ class TestSkopeoImageUploader(base.TestCase):
'add-foo-plugin',
{'foo_version': '1.0.1'},
False,
'full')
'full',
None)
)
mock_ansible.assert_not_called()

20
tripleo_common/tests/image/test_kolla_builder.py
View File

@ -840,6 +840,7 @@ class TestPrepare(base.TestCase):
env = {
'parameter_defaults': {
'LocalContainerRegistry': '192.0.2.1',
'DockerRegistryMirror': 'http://192.0.2.2/reg/',
'ContainerImagePrepare': [{
'set': mapping_args,
'tag_from_label': 'foo',
@ -896,7 +897,10 @@ class TestPrepare(base.TestCase):
append_tag=mock.ANY,
modify_role=None,
modify_only_with_labels=None,
modify_vars=None
modify_vars=None,
mirrors={
'docker.io': 'http://192.0.2.2/reg/'
}
),
mock.call(
excludes=['nova', 'neutron'],
@ -911,7 +915,10 @@ class TestPrepare(base.TestCase):
append_tag=mock.ANY,
modify_role='add-foo-plugin',
modify_only_with_labels=['kolla_version'],
modify_vars={'foo_version': '1.0.1'}
modify_vars={'foo_version': '1.0.1'},
mirrors={
'docker.io': 'http://192.0.2.2/reg/'
}
)
])
@ -995,7 +1002,8 @@ class TestPrepare(base.TestCase):
append_tag=mock.ANY,
modify_role=None,
modify_only_with_labels=None,
modify_vars=None
modify_vars=None,
mirrors={}
),
mock.call(
excludes=['nova', 'neutron'],
@ -1010,11 +1018,13 @@ class TestPrepare(base.TestCase):
append_tag=mock.ANY,
modify_role='add-foo-plugin',
modify_only_with_labels=['kolla_version'],
modify_vars={'foo_version': '1.0.1'}
modify_vars={'foo_version': '1.0.1'},
mirrors={}
)
])
mock_im.assert_called_once_with(mock.ANY, dry_run=True, cleanup='full')
mock_im.assert_called_once_with(mock.ANY, dry_run=True, cleanup='full',
mirrors={})
self.assertEqual(
{