openstack/tripleo-common
Code Issues Proposed changes

Add ability to filter container images to modify

Browse Source 
We want to be able to filter out and not attempt modifying any of the
container images that are not originated from the kolla project. This
was previously done by running the container image prepare twice, the
first time with the modify playbook and an exclude list for the images,
the second time without the modify playbook only for the images
previously excluded.  This forced us to maintain a list of the images
to exclude which is not optimal as we add more images. The new approach
relies on the image labels to determine if it can be modified.

Change-Id: Ie7f628bb54c83c78e810320c739afff158234804
This commit is contained in:
Martin André
2018-07-03 18:43:41 +02:00
parent e4fb52ef35
commit 7cfe2af2e0
3 changed files with 57 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
tripleo_common/image/image_uploader.py
View File

@@ -325,6 +325,12 @@ class DockerImageUploader(ImageUploader):
i = DockerImageUploader._inspect(image_url.geturl(), insecure)
return i.get('Digest')
@staticmethod
def _image_labels(image, insecure):
image_url = DockerImageUploader._image_to_url(image)
i = DockerImageUploader._inspect(image_url.geturl(), insecure)
return i.get('Labels', {}) or {}
@staticmethod
def _image_exists(image, insecure_registries):
try:
@@ -446,6 +452,18 @@ class DockerImageUploader(ImageUploader):
return self._discover_tag_from_inspect(i, image, tag_from_label,
fallback_tag)
def filter_images_with_labels(self, images, labels):
images_with_labels = []
for image in images:
url = self._image_to_url(image)
image_labels = self._image_labels(url.geturl(),
self.is_insecure_registry(
url.netloc))
if set(labels).issubset(set(image_labels)):
images_with_labels.append(image)
return images_with_labels
def cleanup(self, local_images):
if not local_images:
return []

50
tripleo_common/image/kolla_builder.py
View File

@@ -138,13 +138,10 @@ def container_images_prepare_multi(environment, roles_data, dry_run=False,
pull_source = cip_entry.get('pull_source')
modify_role = cip_entry.get('modify_role')
modify_vars = cip_entry.get('modify_vars')
if modify_role:
append_tag = cip_entry.get(
'modify_append_tag',
time.strftime('-modified-%Y%m%d%H%M%S')
)
else:
append_tag = None
modify_only_with_labels = cip_entry.get('modify_only_with_labels')
modify_append_tag = cip_entry.get('modify_append_tag',
time.strftime(
'-modified-%Y%m%d%H%M%S'))
prepare_data = container_images_prepare(
excludes=cip_entry.get('excludes'),
@@ -156,9 +153,10 @@ def container_images_prepare_multi(environment, roles_data, dry_run=False,
output_env_file='image_params',
output_images_file='upload_data',
tag_from_label=cip_entry.get('tag_from_label'),
append_tag=append_tag,
modify_append_tag=modify_append_tag,
modify_role=modify_role,
modify_vars=modify_vars,
modify_only_with_labels=modify_only_with_labels,
)
env_params.update(prepare_data['image_params'])
@@ -192,8 +190,8 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
pull_source=None, push_destination=None,
mapping_args=None, output_env_file=None,
output_images_file=None, tag_from_label=None,
append_tag=None, modify_role=None,
modify_vars=None):
modify_append_tag=None, modify_role=None,
modify_vars=None, modify_only_with_labels=None):
"""Perform container image preparation
:param template_file: path to Jinja2 file containing all image entries
@@ -212,18 +210,19 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
:param output_images_file: key to use for image upload data
:param tag_from_label: string when set will trigger tag discovery on every
image
:param append_tag: string to append to the tag for the destination image
:param modify_append_tag: string to append to the tag for the destination
image
:param modify_role: string of ansible role name to run during upload before
the push to destination
:param modify_vars: dict of variables to pass to modify_role
:param modify_only_with_labels: only modify the container images with the
given labels
:returns: dict with entries for the supplied output_env_file or
output_images_file
"""
if mapping_args is None:
mapping_args = {}
if not append_tag:
append_tag = ''
if service_filter:
if 'OS::TripleO::Services::OpenDaylightApi' in service_filter:
@@ -253,9 +252,10 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
result = builder.container_images_from_template(
filter=ffunc, **mapping_args)
uploader = image_uploader.ImageUploadManager().uploader('docker')
images = [i.get('imagename', '') for i in result]
if tag_from_label:
uploader = image_uploader.ImageUploadManager().uploader('docker')
images = [i.get('imagename', '') for i in result]
image_version_tags = uploader.discover_image_tags(
images, tag_from_label)
for entry in result:
@@ -265,9 +265,23 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
entry['imagename'] = '%s:%s' % (
image_no_tag, image_version_tags[image_no_tag])
if modify_only_with_labels:
images_with_labels = uploader.filter_images_with_labels(
images, modify_only_with_labels)
params = {}
for entry in result:
imagename = entry.get('imagename', '')
append_tag = ''
if modify_role and (
(not modify_only_with_labels)
or imagename in images_with_labels):
entry['modify_role'] = modify_role
if modify_append_tag:
entry['modify_append_tag'] = modify_append_tag
append_tag = modify_append_tag
if modify_vars:
entry['modify_vars'] = modify_vars
if pull_source:
entry['pull_source'] = pull_source
if push_destination:
@@ -276,12 +290,6 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
# push_destination, since that is where they will be uploaded to
image = imagename.partition('/')[2]
imagename = '/'.join((push_destination, image))
if append_tag:
entry['modify_append_tag'] = append_tag
if modify_role:
entry['modify_role'] = modify_role
if modify_vars:
entry['modify_vars'] = modify_vars
if 'params' in entry:
for p in entry.pop('params'):
params[p] = imagename + append_tag

14
tripleo_common/tests/image/test_kolla_builder.py
View File

@@ -792,6 +792,7 @@ class TestPrepare(base.TestCase):
'excludes': ['nova', 'neutron'],
'push_destination': '192.0.2.1:8787',
'modify_role': 'add-foo-plugin',
'modify_only_with_labels': ['kolla_version'],
'modify_vars': {'foo_version': '1.0.1'}
}]
}
@@ -834,8 +835,9 @@ class TestPrepare(base.TestCase):
push_destination=None,
service_filter=None,
tag_from_label='foo',
append_tag=None,
modify_append_tag=mock.ANY,
modify_role=None,
modify_only_with_labels=None,
modify_vars=None
),
mock.call(
@@ -848,8 +850,9 @@ class TestPrepare(base.TestCase):
push_destination='192.0.2.1:8787',
service_filter=None,
tag_from_label='bar',
append_tag=mock.ANY,
modify_append_tag=mock.ANY,
modify_role='add-foo-plugin',
modify_only_with_labels=['kolla_version'],
modify_vars={'foo_version': '1.0.1'}
)
])
@@ -887,6 +890,7 @@ class TestPrepare(base.TestCase):
'excludes': ['nova', 'neutron'],
'push_destination': '192.0.2.1:8787',
'modify_role': 'add-foo-plugin',
'modify_only_with_labels': ['kolla_version'],
'modify_vars': {'foo_version': '1.0.1'},
'modify_append_tag': 'modify-123'
}]
@@ -930,8 +934,9 @@ class TestPrepare(base.TestCase):
push_destination=None,
service_filter=None,
tag_from_label='foo',
append_tag=None,
modify_append_tag=mock.ANY,
modify_role=None,
modify_only_with_labels=None,
modify_vars=None
),
mock.call(
@@ -944,8 +949,9 @@ class TestPrepare(base.TestCase):
push_destination='192.0.2.1:8787',
service_filter=None,
tag_from_label='bar',
append_tag=mock.ANY,
modify_append_tag=mock.ANY,
modify_role='add-foo-plugin',
modify_only_with_labels=['kolla_version'],
modify_vars={'foo_version': '1.0.1'}
)
])