diff --git a/tripleo_common/image/image_uploader.py b/tripleo_common/image/image_uploader.py index 04f74b6f9..10b199052 100644 --- a/tripleo_common/image/image_uploader.py +++ b/tripleo_common/image/image_uploader.py @@ -21,10 +21,13 @@ import logging import netifaces import os import requests +import shutil import six from six.moves import urllib import subprocess +import tempfile import tenacity +import yaml import docker try: @@ -32,6 +35,7 @@ try: except ImportError: from docker import Client from oslo_concurrency import processutils +from tripleo_common.actions import ansible from tripleo_common.image.base import BaseImageManager from tripleo_common.image.exception import ImageUploaderException @@ -96,9 +100,13 @@ class ImageUploadManager(BaseImageManager): # This updates the parsed upload_images dict with real values item['push_destination'] = push_destination + append_tag = item.get('append_tag') + modify_role = item.get('modify_role') + modify_vars = item.get('modify_vars') self.uploader(uploader).add_upload_task( - image_name, pull_source, push_destination) + image_name, pull_source, push_destination, + append_tag, modify_role, modify_vars) for uploader in self.uploaders.values(): uploader.run_tasks() @@ -122,7 +130,8 @@ class ImageUploader(object): pass @abc.abstractmethod - def add_upload_task(self, image_name, pull_source, push_destination): + def add_upload_task(self, image_name, pull_source, push_destination, + append_tag, modify_role, modify_vars): """Add an upload task to be executed later""" pass @@ -150,41 +159,90 @@ class DockerImageUploader(ImageUploader): self.secure_registries = set(SECURE_REGISTRIES) self.insecure_registries = set() + @staticmethod + def run_modify_playbook(modify_role, modify_vars, + source_image, target_image, append_tag): + vars = {} + if modify_vars: + vars.update(modify_vars) + vars['source_image'] = source_image + vars['target_image'] = target_image + vars['modified_append_tag'] = append_tag + LOG.debug('Playbook variables: \n%s' % yaml.safe_dump( + vars, default_flow_style=False)) + playbook = [{ + 'hosts': 'localhost', + 'tasks': [{ + 'name': 'Import role %s' % modify_role, + 'import_role': { + 'name': modify_role + }, + 'vars': vars + }] + }] + LOG.debug('Playbook: \n%s' % yaml.safe_dump( + playbook, default_flow_style=False)) + work_dir = tempfile.mkdtemp(prefix='tripleo-modify-image-playbook-') + try: + action = ansible.AnsiblePlaybookAction( + playbook=playbook, + work_dir=work_dir + ) + result = action.run(None) + LOG.debug(result.get('stdout', '')) + + finally: + shutil.rmtree(work_dir) + @staticmethod def upload_image(image_name, pull_source, push_destination, - insecure_registries): + insecure_registries, append_tag, modify_role, + modify_vars): LOG.info('imagename: %s' % image_name) dockerc = Client(base_url='unix://var/run/docker.sock', version='auto') if ':' in image_name: image = image_name.rpartition(':')[0] - tag = image_name.rpartition(':')[2] + source_tag = image_name.rpartition(':')[2] else: image = image_name - tag = 'latest' + source_tag = 'latest' if pull_source: repo = pull_source + '/' + image else: repo = image - full_image = repo + ':' + tag - new_repo = push_destination + '/' + repo.partition('/')[2] - full_new_repo = new_repo + ':' + tag + source_image = repo + ':' + source_tag + target_image_no_tag = push_destination + '/' + repo.partition('/')[2] + append_tag = append_tag or '' + target_tag = source_tag + append_tag + target_image_source_tag = target_image_no_tag + ':' + source_tag + target_image = target_image_no_tag + ':' + target_tag - if DockerImageUploader._images_match(full_image, full_new_repo, + if DockerImageUploader._images_match(source_image, target_image, insecure_registries): LOG.info('Skipping upload for image %s' % image_name) return [] - DockerImageUploader._pull(dockerc, repo, tag=tag) + DockerImageUploader._pull(dockerc, repo, tag=source_tag) - response = dockerc.tag(image=full_image, repository=new_repo, - tag=tag, force=True) - LOG.debug(response) + if modify_role: + DockerImageUploader.run_modify_playbook( + modify_role, modify_vars, source_image, + target_image_source_tag, append_tag) + # raise an exception if the playbook didn't tag + # the expected target image + dockerc.inspect_image(target_image) + else: + response = dockerc.tag( + image=source_image, repository=target_image_no_tag, + tag=target_tag, force=True + ) + LOG.debug(response) - DockerImageUploader._push(dockerc, new_repo, tag=tag) + DockerImageUploader._push(dockerc, target_image_no_tag, tag=target_tag) LOG.info('Completed upload for image %s' % image_name) - return full_image, full_new_repo + return source_image, target_image @staticmethod @tenacity.retry( # Retry up to 5 times with jittered exponential backoff @@ -353,7 +411,8 @@ class DockerImageUploader(ImageUploader): else: LOG.warning(e) - def add_upload_task(self, image_name, pull_source, push_destination): + def add_upload_task(self, image_name, pull_source, push_destination, + append_tag, modify_role, modify_vars): # prime self.insecure_registries if pull_source: self.is_insecure_registry(self._image_to_url(pull_source).netloc) @@ -361,7 +420,8 @@ class DockerImageUploader(ImageUploader): self.is_insecure_registry(self._image_to_url(image_name).netloc) self.is_insecure_registry(self._image_to_url(push_destination).netloc) self.upload_tasks.append((image_name, pull_source, push_destination, - self.insecure_registries)) + self.insecure_registries, append_tag, + modify_role, modify_vars)) def run_tasks(self): if not self.upload_tasks: diff --git a/tripleo_common/image/kolla_builder.py b/tripleo_common/image/kolla_builder.py index cfdd8f20a..a6afc59e4 100644 --- a/tripleo_common/image/kolla_builder.py +++ b/tripleo_common/image/kolla_builder.py @@ -21,6 +21,7 @@ import re import subprocess import sys import tempfile +import time import yaml from tripleo_common.image import base @@ -114,11 +115,18 @@ def container_images_prepare_multi(environment, roles_data): env_params = {} service_filter = build_service_filter(environment, roles_data) + modified_timestamp = time.strftime('-modified-%Y%m%d%H%M%S') for cip_entry in cip: mapping_args = cip_entry.get('set') push_destination = cip_entry.get('push_destination') pull_source = cip_entry.get('pull_source') + modify_role = cip_entry.get('modify_role') + modify_vars = cip_entry.get('modify_vars') + if modify_role: + append_tag = modified_timestamp + else: + append_tag = None prepare_data = container_images_prepare( excludes=cip_entry.get('excludes'), @@ -129,16 +137,21 @@ def container_images_prepare_multi(environment, roles_data): output_env_file='image_params', output_images_file='upload_data', tag_from_label=cip_entry.get('tag_from_label'), + append_tag=append_tag, + modify_role=modify_role, + modify_vars=modify_vars ) env_params.update(prepare_data['image_params']) - if push_destination or pull_source: + if push_destination or pull_source or modify_role: with tempfile.NamedTemporaryFile(mode='w') as f: yaml.safe_dump({ 'container_images': prepare_data['upload_data'] }, f) uploader = image_uploader.ImageUploadManager( - [f.name], verbose=True) + [f.name], + verbose=True, + ) uploader.upload() return env_params @@ -157,7 +170,9 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE, excludes=None, service_filter=None, pull_source=None, push_destination=None, mapping_args=None, output_env_file=None, - output_images_file=None, tag_from_label=None): + output_images_file=None, tag_from_label=None, + append_tag=None, modify_role=None, + modify_vars=None): """Perform container image preparation :param template_file: path to Jinja2 file containing all image entries @@ -174,12 +189,18 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE, :param output_images_file: key to use for image upload data :param tag_from_label: string when set will trigger tag discovery on every image + :param append_tag: string to append to the tag for the destination image + :param modify_role: string of ansible role name to run during upload before + the push to destination + :param modify_vars: dict of variables to pass to modify_role :returns: dict with entries for the supplied output_env_file or output_images_file """ if mapping_args is None: mapping_args = {} + if not append_tag: + append_tag = '' if service_filter: if 'OS::TripleO::Services::OpenDaylightApi' in service_filter: @@ -227,9 +248,15 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE, # push_destination, since that is where they will be uploaded to image = imagename.partition('/')[2] imagename = '/'.join((push_destination, image)) + if append_tag: + entry['append_tag'] = append_tag + if modify_role: + entry['modify_role'] = modify_role + if modify_vars: + entry['modify_vars'] = modify_vars if 'params' in entry: for p in entry.pop('params'): - params[p] = imagename + params[p] = imagename + append_tag if 'services' in entry: del(entry['services']) diff --git a/tripleo_common/tests/image/test_image_uploader.py b/tripleo_common/tests/image/test_image_uploader.py index b694c7eba..5f5a1834f 100644 --- a/tripleo_common/tests/image/test_image_uploader.py +++ b/tripleo_common/tests/image/test_image_uploader.py @@ -20,6 +20,7 @@ import requests import six import urllib3 +from oslo_concurrency import processutils from tripleo_common.image.exception import ImageUploaderException from tripleo_common.image import image_uploader from tripleo_common.tests import base @@ -164,7 +165,10 @@ class TestDockerImageUploader(base.TestCase): self.uploader.upload_image(image + ':' + tag, None, push_destination, - set()) + set(), + None, + None, + None) self.dockermock.assert_called_once_with( base_url='unix://var/run/docker.sock', version='auto') @@ -189,7 +193,10 @@ class TestDockerImageUploader(base.TestCase): self.uploader.upload_image(image, None, push_destination, - set()) + set(), + None, + None, + None) self.dockermock.assert_called_once_with( base_url='unix://var/run/docker.sock', version='auto') @@ -220,7 +227,10 @@ class TestDockerImageUploader(base.TestCase): self.uploader.upload_image(image + ':' + tag, None, push_destination, - set()) + set(), + None, + None, + None) self.dockermock.assert_called_once_with( base_url='unix://var/run/docker.sock', version='auto') @@ -230,6 +240,109 @@ class TestDockerImageUploader(base.TestCase): self.dockermock.return_value.tag.assert_not_called() self.dockermock.return_value.push.assert_not_called() + @mock.patch('subprocess.Popen') + @mock.patch('tripleo_common.actions.' + 'ansible.AnsiblePlaybookAction', autospec=True) + def test_modify_upload_image(self, mock_ansible, mock_popen): + result1 = { + 'Digest': 'a' + } + result2 = { + 'Digest': 'b' + } + mock_process = mock.Mock() + mock_process.communicate.side_effect = [ + (json.dumps(result1), ''), + (json.dumps(result2), ''), + ] + + mock_process.returncode = 0 + mock_popen.return_value = mock_process + + image = 'docker.io/tripleomaster/heat-docker-agents-centos' + tag = 'latest' + append_tag = 'modify-123' + push_destination = 'localhost:8787' + push_image = 'localhost:8787/tripleomaster/heat-docker-agents-centos' + playbook = [{ + 'tasks': [{ + 'import_role': { + 'name': 'add-foo-plugin' + }, + 'name': 'Import role add-foo-plugin', + 'vars': { + 'target_image': '%s:%s' % (push_image, tag), + 'modified_append_tag': append_tag, + 'source_image': '%s:%s' % (image, tag), + 'foo_version': '1.0.1' + } + }], + 'hosts': 'localhost' + }] + + self.uploader.upload_image(image + ':' + tag, + None, + push_destination, + set(), + append_tag, + 'add-foo-plugin', + {'foo_version': '1.0.1'}) + + self.dockermock.assert_called_once_with( + base_url='unix://var/run/docker.sock', version='auto') + + self.dockermock.return_value.pull.assert_called_once_with( + image, tag=tag, stream=True) + mock_ansible.assert_called_once_with( + playbook=playbook, work_dir=mock.ANY) + self.dockermock.return_value.tag.assert_not_called() + self.dockermock.return_value.push.assert_called_once_with( + push_image, + tag=tag + append_tag, + stream=True) + + @mock.patch('subprocess.Popen') + @mock.patch('tripleo_common.actions.' + 'ansible.AnsiblePlaybookAction', autospec=True) + def test_modify_image_failed(self, mock_ansible, mock_popen): + result1 = { + 'Digest': 'a' + } + result2 = { + 'Digest': 'b' + } + mock_process = mock.Mock() + mock_process.communicate.side_effect = [ + (json.dumps(result1), ''), + (json.dumps(result2), ''), + ] + + mock_process.returncode = 0 + mock_popen.return_value = mock_process + + image = 'docker.io/tripleomaster/heat-docker-agents-centos' + tag = 'latest' + append_tag = 'modify-123' + push_destination = 'localhost:8787' + error = processutils.ProcessExecutionError( + '', 'ouch', -1, 'ansible-playbook') + mock_ansible.return_value.run.side_effect = error + + self.assertRaises( + processutils.ProcessExecutionError, + self.uploader.upload_image, + image + ':' + tag, None, push_destination, set(), append_tag, + 'add-foo-plugin', {'foo_version': '1.0.1'} + ) + + self.dockermock.assert_called_once_with( + base_url='unix://var/run/docker.sock', version='auto') + + self.dockermock.return_value.pull.assert_called_once_with( + image, tag=tag, stream=True) + self.dockermock.return_value.tag.assert_not_called() + self.dockermock.return_value.push.assert_not_called() + @mock.patch('requests.get') def test_is_insecure_registry_known(self, mock_get): self.assertFalse( diff --git a/tripleo_common/tests/image/test_kolla_builder.py b/tripleo_common/tests/image/test_kolla_builder.py index 84478b3c0..2ef6a806b 100644 --- a/tripleo_common/tests/image/test_kolla_builder.py +++ b/tripleo_common/tests/image/test_kolla_builder.py @@ -669,7 +669,9 @@ class TestPrepare(base.TestCase): 'set': mapping_args, 'tag_from_label': 'bar', 'excludes': ['nova', 'neutron'], - 'push_destination': '192.0.2.1:8787' + 'push_destination': '192.0.2.1:8787', + 'modify_role': 'add-foo-plugin', + 'modify_vars': {'foo_version': '1.0.1'} }] } } @@ -709,7 +711,10 @@ class TestPrepare(base.TestCase): pull_source=None, push_destination=None, service_filter=None, - tag_from_label='foo' + tag_from_label='foo', + append_tag=None, + modify_role=None, + modify_vars=None ), mock.call( excludes=['nova', 'neutron'], @@ -719,7 +724,10 @@ class TestPrepare(base.TestCase): pull_source=None, push_destination='192.0.2.1:8787', service_filter=None, - tag_from_label='bar' + tag_from_label='bar', + append_tag=mock.ANY, + modify_role='add-foo-plugin', + modify_vars={'foo_version': '1.0.1'} ) ])