openstack
/
tripleo-common
Code Issues Proposed changes

Merge "Use ContainerImageRegistryCredentials for registry auth"

This commit is contained in:
Zuul 2019-04-16 00:47:08 +00:00 committed by Gerrit Code Review
parent 2c5d3f5f7d a47e0129e6
commit 276aba59e9
4 changed files with 234 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
tripleo_common/image/image_uploader.py
View File

@ -109,7 +109,7 @@ class ImageUploadManager(BaseImageManager):
def __init__(self, config_files=None,
dry_run=False, cleanup=CLEANUP_FULL,
mirrors=None):
mirrors=None, registry_credentials=None):
if config_files is None:
config_files = []
super(ImageUploadManager, self).__init__(config_files)
@ -122,6 +122,26 @@ class ImageUploadManager(BaseImageManager):
if mirrors:
for uploader in self.uploaders.values():
uploader.mirrors.update(mirrors)
if registry_credentials:
self.validate_registry_credentials(registry_credentials)
for uploader in self.uploaders.values():
uploader.registry_credentials = registry_credentials
def validate_registry_credentials(self, creds_data):
if not isinstance(creds_data, dict):
raise TypeError('Credentials data must be a dict')
for registry, cred_entry in creds_data.items():
if not isinstance(cred_entry, dict) or len(cred_entry) != 1:
raise TypeError('Credentials entry must be'
'a dict with a single item')
if not isinstance(registry, six.string_types):
raise TypeError('Key must be a registry host string: %s' %
registry)
username, password = next(iter(cred_entry.items()))
if not (isinstance(username, six.string_types) and
isinstance(password, six.string_types)):
raise TypeError('Username and password must be strings: %s' %
username)
def discover_image_tag(self, image, tag_from_label=None,
username=None, password=None):
@ -197,6 +217,7 @@ class BaseImageUploader(object):
# A mapping of layer hashs to the image which first copied that
# layer to the target
self.image_layers = {}
self.registry_credentials = {}
@classmethod
def init_registries_cache(cls):
@ -215,6 +236,13 @@ class BaseImageUploader(object):
def run_tasks(self):
pass
def credentials_for_registry(self, registry):
creds = self.registry_credentials.get(registry)
if not creds:
return None, None
username, password = next(iter(creds.items()))
return username, password
@classmethod
def run_modify_playbook(cls, modify_role, modify_vars,
source_image, target_image, append_tag,
@ -707,8 +735,13 @@ class SkopeoImageUploader(BaseImageUploader):
if t.dry_run:
return []
target_username, target_password = self.credentials_for_registry(
t.target_image_url.netloc)
target_session = self.authenticate(
t.target_image_url)
t.target_image_url,
username=target_username,
password=target_password
)
if t.modify_role and self._image_exists(
t.target_image, target_session):
@ -716,8 +749,13 @@ class SkopeoImageUploader(BaseImageUploader):
t.target_image)
return []
source_username, source_password = self.credentials_for_registry(
t.source_image_url.netloc)
source_session = self.authenticate(
t.source_image_url)
t.source_image_url,
username=source_username,
password=source_password
)
source_inspect = self._inspect(
t.source_image_url,
@ -867,8 +905,12 @@ class PythonImageUploader(BaseImageUploader):
if t.dry_run:
return []
target_username, target_password = self.credentials_for_registry(
t.target_image_url.netloc)
target_session = self.authenticate(
t.target_image_url
t.target_image_url,
username=target_username,
password=target_password
)
self._detect_target_export(t.target_image_url, target_session)
@ -883,8 +925,12 @@ class PythonImageUploader(BaseImageUploader):
else:
copy_target_url = t.target_image_url
source_username, source_password = self.credentials_for_registry(
t.source_image_url.netloc)
source_session = self.authenticate(
t.source_image_url
t.source_image_url,
username=source_username,
password=source_password
)
manifest_str = self._fetch_manifest(
@ -1639,7 +1685,9 @@ def upload_task(args):
def discover_tag_from_inspect(args):
self, image, tag_from_label = args
image_url = self._image_to_url(image)
session = self.authenticate(image_url)
username, password = self.credentials_for_registry(image_url.netloc)
session = self.authenticate(
image_url, username=username, password=password)
i = self._inspect(image_url, session=session)
if ':' in image_url.path:
# break out the tag from the url to be the fallback tag

18
tripleo_common/image/kolla_builder.py
View File

@ -150,6 +150,8 @@ def container_images_prepare_multi(environment, roles_data, dry_run=False,
if mirror:
mirrors['docker.io'] = mirror
creds = pd.get('ContainerImageRegistryCredentials')
env_params = {}
service_filter = build_service_filter(environment, roles_data)
@ -185,7 +187,8 @@ def container_images_prepare_multi(environment, roles_data, dry_run=False,
modify_role=modify_role,
modify_vars=modify_vars,
modify_only_with_labels=modify_only_with_labels,
mirrors=mirrors
mirrors=mirrors,
registry_credentials=creds
)
env_params.update(prepare_data['image_params'])
@ -198,7 +201,8 @@ def container_images_prepare_multi(environment, roles_data, dry_run=False,
[f.name],
dry_run=dry_run,
cleanup=cleanup,
mirrors=mirrors
mirrors=mirrors,
registry_credentials=creds
)
uploader.upload()
return env_params
@ -221,7 +225,7 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
output_images_file=None, tag_from_label=None,
append_tag=None, modify_role=None,
modify_vars=None, modify_only_with_labels=None,
mirrors=None):
mirrors=None, registry_credentials=None):
"""Perform container image preparation
:param template_file: path to Jinja2 file containing all image entries
@ -248,6 +252,11 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
:param modify_only_with_labels: only modify the container images with the
given labels
:param mirrors: dict of registry netloc values to mirror urls
:param registry_credentials: dict of registry netloc values to
authentication credentials for that registry.
The value is a single-entry dict where the
username is the key and the password is the
value.
:returns: dict with entries for the supplied output_env_file or
output_images_file
"""
@ -277,7 +286,8 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
result = builder.container_images_from_template(
filter=ffunc, **mapping_args)
manager = image_uploader.ImageUploadManager(mirrors=mirrors)
manager = image_uploader.ImageUploadManager(
mirrors=mirrors, registry_credentials=registry_credentials)
uploader = manager.uploader('python')
images = [i.get('imagename', '') for i in result]

158
tripleo_common/tests/image/test_image_uploader.py
View File

@ -154,6 +154,69 @@ class TestImageUploadManager(base.TestCase):
manager.get_uploader,
'unknown')
def test_validate_registry_credentials(self):
# valid credentials
image_uploader.ImageUploadManager(
self.filelist,
registry_credentials=None)
image_uploader.ImageUploadManager(
self.filelist,
registry_credentials={})
manager = image_uploader.ImageUploadManager(
self.filelist,
registry_credentials={
'docker.io': {'my_username': 'my_password'},
u'quay.io': {u'quay_username': u'quay_password'},
})
self.assertEqual(
('my_username', 'my_password'),
manager.uploader('python').credentials_for_registry('docker.io')
)
self.assertEqual(
('quay_username', 'quay_password'),
manager.uploader('python').credentials_for_registry('quay.io')
)
# invalid credentials
self.assertRaises(
TypeError,
image_uploader.ImageUploadManager,
self.filelist,
registry_credentials='foo'
)
self.assertRaises(
TypeError,
image_uploader.ImageUploadManager,
self.filelist,
registry_credentials={
1234: {'my_username': 'my_password'},
}
)
self.assertRaises(
TypeError,
image_uploader.ImageUploadManager,
self.filelist,
registry_credentials={
'docker.io': {True: 'my_password'},
}
)
self.assertRaises(
TypeError,
image_uploader.ImageUploadManager,
self.filelist,
registry_credentials={
'docker.io': {'my_username': True},
}
)
self.assertRaises(
TypeError,
image_uploader.ImageUploadManager,
self.filelist,
registry_credentials={
'docker.io': {'my_username': 'my_password', 'foo': 'bar'},
}
)
class TestBaseImageUploader(base.TestCase):
@ -1100,10 +1163,14 @@ class TestPythonImageUploader(base.TestCase):
authenticate.assert_has_calls([
mock.call(
target_url
target_url,
username=None,
password=None
),
mock.call(
source_url
source_url,
username=None,
password=None
),
])
@ -1128,6 +1195,77 @@ class TestPythonImageUploader(base.TestCase):
target_session=target_session
)
@mock.patch('tripleo_common.image.image_uploader.'
'PythonImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'PythonImageUploader._fetch_manifest')
@mock.patch('tripleo_common.image.image_uploader.'
'PythonImageUploader._cross_repo_mount')
@mock.patch('tripleo_common.image.image_uploader.'
'PythonImageUploader._copy_registry_to_registry')
def test_authenticate_upload_image(
self, _copy_registry_to_registry, _cross_repo_mount,
_fetch_manifest, authenticate):
self.uploader.registry_credentials = {
'docker.io': {'my_username': 'my_password'},
'localhost:8787': {'local_username': 'local_password'},
}
target_session = mock.Mock()
source_session = mock.Mock()
authenticate.side_effect = [
target_session,
source_session
]
manifest = json.dumps({
'config': {
'digest': 'sha256:1234',
},
'layers': [
{'digest': 'sha256:aaa'},
{'digest': 'sha256:bbb'},
{'digest': 'sha256:ccc'}
],
})
_fetch_manifest.return_value = manifest
image = 'docker.io/tripleomaster/heat-docker-agents-centos'
tag = 'latest'
push_destination = 'localhost:8787'
# push_image = 'localhost:8787/tripleomaster/heat-docker-agents-centos'
task = image_uploader.UploadTask(
image_name=image + ':' + tag,
pull_source=None,
push_destination=push_destination,
append_tag=None,
modify_role=None,
modify_vars=None,
dry_run=False,
cleanup='full'
)
self.assertEqual(
[],
self.uploader.upload_image(task)
)
source_url = urlparse('docker://docker.io/tripleomaster/'
'heat-docker-agents-centos:latest')
target_url = urlparse('docker://localhost:8787/tripleomaster/'
'heat-docker-agents-centos:latest')
authenticate.assert_has_calls([
mock.call(
target_url,
username='local_username',
password='local_password'
),
mock.call(
source_url,
username='my_username',
password='my_password'
),
])
@mock.patch('tripleo_common.image.image_uploader.'
'PythonImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
@ -1183,10 +1321,14 @@ class TestPythonImageUploader(base.TestCase):
authenticate.assert_has_calls([
mock.call(
target_url
target_url,
username=None,
password=None
),
mock.call(
source_url
source_url,
username=None,
password=None
),
])
@ -1268,10 +1410,14 @@ class TestPythonImageUploader(base.TestCase):
)
authenticate.assert_has_calls([
mock.call(
target_url
target_url,
username=None,
password=None
),
mock.call(
source_url
source_url,
username=None,
password=None
),
])

17
tripleo_common/tests/image/test_kolla_builder.py
View File

@ -1039,6 +1039,9 @@ class TestPrepare(base.TestCase):
'parameter_defaults': {
'LocalContainerRegistry': '192.0.2.1',
'DockerRegistryMirror': 'http://192.0.2.2/reg/',
'ContainerImageRegistryCredentials': {
'docker.io': {'my_username': 'my_password'}
},
'ContainerImagePrepare': [{
'set': mapping_args,
'tag_from_label': 'foo',
@ -1098,6 +1101,9 @@ class TestPrepare(base.TestCase):
modify_vars=None,
mirrors={
'docker.io': 'http://192.0.2.2/reg/'
},
registry_credentials={
'docker.io': {'my_username': 'my_password'}
}
),
mock.call(
@ -1116,6 +1122,9 @@ class TestPrepare(base.TestCase):
modify_vars={'foo_version': '1.0.1'},
mirrors={
'docker.io': 'http://192.0.2.2/reg/'
},
registry_credentials={
'docker.io': {'my_username': 'my_password'}
}
)
])
@ -1201,7 +1210,8 @@ class TestPrepare(base.TestCase):
modify_role=None,
modify_only_with_labels=None,
modify_vars=None,
mirrors={}
mirrors={},
registry_credentials=None
),
mock.call(
excludes=['nova', 'neutron'],
@ -1217,12 +1227,13 @@ class TestPrepare(base.TestCase):
modify_role='add-foo-plugin',
modify_only_with_labels=['kolla_version'],
modify_vars={'foo_version': '1.0.1'},
mirrors={}
mirrors={},
registry_credentials=None
)
])
mock_im.assert_called_once_with(mock.ANY, dry_run=True, cleanup='full',
mirrors={})
mirrors={}, registry_credentials=None)
self.assertEqual(
{