Merge "Implement image customization during upload"

tripleo_common/image/image_uploader.py

@@ -21,10 +21,13 @@ import logging
 import netifaces
 import os
 import requests
+import shutil
 import six
 from six.moves import urllib
 import subprocess
+import tempfile
 import tenacity
+import yaml
 
 import docker
 try:
@@ -32,6 +35,7 @@ try:
 except ImportError:
     from docker import Client
 from oslo_concurrency import processutils
+from tripleo_common.actions import ansible
 from tripleo_common.image.base import BaseImageManager
 from tripleo_common.image.exception import ImageUploaderException
 
@@ -96,9 +100,13 @@ class ImageUploadManager(BaseImageManager):
 
             # This updates the parsed upload_images dict with real values
             item['push_destination'] = push_destination
+            append_tag = item.get('append_tag')
+            modify_role = item.get('modify_role')
+            modify_vars = item.get('modify_vars')
 
             self.uploader(uploader).add_upload_task(
-                image_name, pull_source, push_destination)
+                image_name, pull_source, push_destination,
+                append_tag, modify_role, modify_vars)
 
         for uploader in self.uploaders.values():
             uploader.run_tasks()
@@ -122,7 +130,8 @@ class ImageUploader(object):
         pass
 
     @abc.abstractmethod
-    def add_upload_task(self, image_name, pull_source, push_destination):
+    def add_upload_task(self, image_name, pull_source, push_destination,
+                        append_tag, modify_role, modify_vars):
         """Add an upload task to be executed later"""
         pass
 
@@ -150,41 +159,90 @@ class DockerImageUploader(ImageUploader):
         self.secure_registries = set(SECURE_REGISTRIES)
         self.insecure_registries = set()
 
+    @staticmethod
+    def run_modify_playbook(modify_role, modify_vars,
+                            source_image, target_image, append_tag):
+        vars = {}
+        if modify_vars:
+            vars.update(modify_vars)
+        vars['source_image'] = source_image
+        vars['target_image'] = target_image
+        vars['modified_append_tag'] = append_tag
+        LOG.debug('Playbook variables: \n%s' % yaml.safe_dump(
+            vars, default_flow_style=False))
+        playbook = [{
+            'hosts': 'localhost',
+            'tasks': [{
+                'name': 'Import role %s' % modify_role,
+                'import_role': {
+                    'name': modify_role
+                },
+                'vars': vars
+            }]
+        }]
+        LOG.debug('Playbook: \n%s' % yaml.safe_dump(
+            playbook, default_flow_style=False))
+        work_dir = tempfile.mkdtemp(prefix='tripleo-modify-image-playbook-')
+        try:
+            action = ansible.AnsiblePlaybookAction(
+                playbook=playbook,
+                work_dir=work_dir
+            )
+            result = action.run(None)
+            LOG.debug(result.get('stdout', ''))
+
+        finally:
+            shutil.rmtree(work_dir)
+
     @staticmethod
     def upload_image(image_name, pull_source, push_destination,
-                     insecure_registries):
+                     insecure_registries, append_tag, modify_role,
+                     modify_vars):
         LOG.info('imagename: %s' % image_name)
         dockerc = Client(base_url='unix://var/run/docker.sock', version='auto')
         if ':' in image_name:
             image = image_name.rpartition(':')[0]
-            tag = image_name.rpartition(':')[2]
+            source_tag = image_name.rpartition(':')[2]
         else:
             image = image_name
-            tag = 'latest'
+            source_tag = 'latest'
         if pull_source:
             repo = pull_source + '/' + image
         else:
             repo = image
 
-        full_image = repo + ':' + tag
+        source_image = repo + ':' + source_tag
-        new_repo = push_destination + '/' + repo.partition('/')[2]
+        target_image_no_tag = push_destination + '/' + repo.partition('/')[2]
-        full_new_repo = new_repo + ':' + tag
+        append_tag = append_tag or ''
+        target_tag = source_tag + append_tag
+        target_image_source_tag = target_image_no_tag + ':' + source_tag
+        target_image = target_image_no_tag + ':' + target_tag
 
-        if DockerImageUploader._images_match(full_image, full_new_repo,
+        if DockerImageUploader._images_match(source_image, target_image,
                                              insecure_registries):
             LOG.info('Skipping upload for image %s' % image_name)
             return []
 
-        DockerImageUploader._pull(dockerc, repo, tag=tag)
+        DockerImageUploader._pull(dockerc, repo, tag=source_tag)
 
-        response = dockerc.tag(image=full_image, repository=new_repo,
+        if modify_role:
-                               tag=tag, force=True)
+            DockerImageUploader.run_modify_playbook(
-        LOG.debug(response)
+                modify_role, modify_vars, source_image,
+                target_image_source_tag, append_tag)
+            # raise an exception if the playbook didn't tag
+            # the expected target image
+            dockerc.inspect_image(target_image)
+        else:
+            response = dockerc.tag(
+                image=source_image, repository=target_image_no_tag,
+                tag=target_tag, force=True
+            )
+            LOG.debug(response)
 
-        DockerImageUploader._push(dockerc, new_repo, tag=tag)
+        DockerImageUploader._push(dockerc, target_image_no_tag, tag=target_tag)
 
         LOG.info('Completed upload for image %s' % image_name)
-        return full_image, full_new_repo
+        return source_image, target_image
 
     @staticmethod
     @tenacity.retry(  # Retry up to 5 times with jittered exponential backoff
@@ -353,7 +411,8 @@ class DockerImageUploader(ImageUploader):
                 else:
                     LOG.warning(e)
 
-    def add_upload_task(self, image_name, pull_source, push_destination):
+    def add_upload_task(self, image_name, pull_source, push_destination,
+                        append_tag, modify_role, modify_vars):
         # prime self.insecure_registries
         if pull_source:
             self.is_insecure_registry(self._image_to_url(pull_source).netloc)
@@ -361,7 +420,8 @@ class DockerImageUploader(ImageUploader):
             self.is_insecure_registry(self._image_to_url(image_name).netloc)
         self.is_insecure_registry(self._image_to_url(push_destination).netloc)
         self.upload_tasks.append((image_name, pull_source, push_destination,
-                                  self.insecure_registries))
+                                  self.insecure_registries, append_tag,
+                                  modify_role, modify_vars))
 
     def run_tasks(self):
         if not self.upload_tasks:

tripleo_common/image/kolla_builder.py

@@ -21,6 +21,7 @@ import re
 import subprocess
 import sys
 import tempfile
+import time
 import yaml
 
 from tripleo_common.image import base
@@ -114,11 +115,18 @@ def container_images_prepare_multi(environment, roles_data):
 
     env_params = {}
     service_filter = build_service_filter(environment, roles_data)
+    modified_timestamp = time.strftime('-modified-%Y%m%d%H%M%S')
 
     for cip_entry in cip:
         mapping_args = cip_entry.get('set')
         push_destination = cip_entry.get('push_destination')
         pull_source = cip_entry.get('pull_source')
+        modify_role = cip_entry.get('modify_role')
+        modify_vars = cip_entry.get('modify_vars')
+        if modify_role:
+            append_tag = modified_timestamp
+        else:
+            append_tag = None
 
         prepare_data = container_images_prepare(
             excludes=cip_entry.get('excludes'),
@@ -129,16 +137,21 @@ def container_images_prepare_multi(environment, roles_data):
             output_env_file='image_params',
             output_images_file='upload_data',
             tag_from_label=cip_entry.get('tag_from_label'),
+            append_tag=append_tag,
+            modify_role=modify_role,
+            modify_vars=modify_vars
         )
         env_params.update(prepare_data['image_params'])
 
-        if push_destination or pull_source:
+        if push_destination or pull_source or modify_role:
             with tempfile.NamedTemporaryFile(mode='w') as f:
                 yaml.safe_dump({
                     'container_images': prepare_data['upload_data']
                 }, f)
                 uploader = image_uploader.ImageUploadManager(
-                    [f.name], verbose=True)
+                    [f.name],
+                    verbose=True,
+                )
                 uploader.upload()
     return env_params
 
@@ -157,7 +170,9 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
                              excludes=None, service_filter=None,
                              pull_source=None, push_destination=None,
                              mapping_args=None, output_env_file=None,
-                             output_images_file=None, tag_from_label=None):
+                             output_images_file=None, tag_from_label=None,
+                             append_tag=None, modify_role=None,
+                             modify_vars=None):
     """Perform container image preparation
 
     :param template_file: path to Jinja2 file containing all image entries
@@ -174,12 +189,18 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
     :param output_images_file: key to use for image upload data
     :param tag_from_label: string when set will trigger tag discovery on every
                            image
+    :param append_tag: string to append to the tag for the destination image
+    :param modify_role: string of ansible role name to run during upload before
+                        the push to destination
+    :param modify_vars: dict of variables to pass to modify_role
     :returns: dict with entries for the supplied output_env_file or
               output_images_file
     """
 
     if mapping_args is None:
         mapping_args = {}
+    if not append_tag:
+        append_tag = ''
 
     if service_filter:
         if 'OS::TripleO::Services::OpenDaylightApi' in service_filter:
@@ -227,9 +248,15 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
             # push_destination, since that is where they will be uploaded to
             image = imagename.partition('/')[2]
             imagename = '/'.join((push_destination, image))
+        if append_tag:
+            entry['append_tag'] = append_tag
+        if modify_role:
+            entry['modify_role'] = modify_role
+        if modify_vars:
+            entry['modify_vars'] = modify_vars
         if 'params' in entry:
             for p in entry.pop('params'):
-                params[p] = imagename
+                params[p] = imagename + append_tag
         if 'services' in entry:
             del(entry['services'])
 

tripleo_common/tests/image/test_image_uploader.py

@@ -20,6 +20,7 @@ import requests
 import six
 import urllib3
 
+from oslo_concurrency import processutils
 from tripleo_common.image.exception import ImageUploaderException
 from tripleo_common.image import image_uploader
 from tripleo_common.tests import base
@@ -164,7 +165,10 @@ class TestDockerImageUploader(base.TestCase):
         self.uploader.upload_image(image + ':' + tag,
                                    None,
                                    push_destination,
-                                   set())
+                                   set(),
+                                   None,
+                                   None,
+                                   None)
 
         self.dockermock.assert_called_once_with(
             base_url='unix://var/run/docker.sock', version='auto')
@@ -189,7 +193,10 @@ class TestDockerImageUploader(base.TestCase):
         self.uploader.upload_image(image,
                                    None,
                                    push_destination,
-                                   set())
+                                   set(),
+                                   None,
+                                   None,
+                                   None)
 
         self.dockermock.assert_called_once_with(
             base_url='unix://var/run/docker.sock', version='auto')
@@ -220,7 +227,10 @@ class TestDockerImageUploader(base.TestCase):
         self.uploader.upload_image(image + ':' + tag,
                                    None,
                                    push_destination,
-                                   set())
+                                   set(),
+                                   None,
+                                   None,
+                                   None)
 
         self.dockermock.assert_called_once_with(
             base_url='unix://var/run/docker.sock', version='auto')
@@ -230,6 +240,109 @@ class TestDockerImageUploader(base.TestCase):
         self.dockermock.return_value.tag.assert_not_called()
         self.dockermock.return_value.push.assert_not_called()
 
+    @mock.patch('subprocess.Popen')
+    @mock.patch('tripleo_common.actions.'
+                'ansible.AnsiblePlaybookAction', autospec=True)
+    def test_modify_upload_image(self, mock_ansible, mock_popen):
+        result1 = {
+            'Digest': 'a'
+        }
+        result2 = {
+            'Digest': 'b'
+        }
+        mock_process = mock.Mock()
+        mock_process.communicate.side_effect = [
+            (json.dumps(result1), ''),
+            (json.dumps(result2), ''),
+        ]
+
+        mock_process.returncode = 0
+        mock_popen.return_value = mock_process
+
+        image = 'docker.io/tripleomaster/heat-docker-agents-centos'
+        tag = 'latest'
+        append_tag = 'modify-123'
+        push_destination = 'localhost:8787'
+        push_image = 'localhost:8787/tripleomaster/heat-docker-agents-centos'
+        playbook = [{
+            'tasks': [{
+                'import_role': {
+                    'name': 'add-foo-plugin'
+                },
+                'name': 'Import role add-foo-plugin',
+                'vars': {
+                    'target_image': '%s:%s' % (push_image, tag),
+                    'modified_append_tag': append_tag,
+                    'source_image': '%s:%s' % (image, tag),
+                    'foo_version': '1.0.1'
+                }
+            }],
+            'hosts': 'localhost'
+        }]
+
+        self.uploader.upload_image(image + ':' + tag,
+                                   None,
+                                   push_destination,
+                                   set(),
+                                   append_tag,
+                                   'add-foo-plugin',
+                                   {'foo_version': '1.0.1'})
+
+        self.dockermock.assert_called_once_with(
+            base_url='unix://var/run/docker.sock', version='auto')
+
+        self.dockermock.return_value.pull.assert_called_once_with(
+            image, tag=tag, stream=True)
+        mock_ansible.assert_called_once_with(
+            playbook=playbook, work_dir=mock.ANY)
+        self.dockermock.return_value.tag.assert_not_called()
+        self.dockermock.return_value.push.assert_called_once_with(
+            push_image,
+            tag=tag + append_tag,
+            stream=True)
+
+    @mock.patch('subprocess.Popen')
+    @mock.patch('tripleo_common.actions.'
+                'ansible.AnsiblePlaybookAction', autospec=True)
+    def test_modify_image_failed(self, mock_ansible, mock_popen):
+        result1 = {
+            'Digest': 'a'
+        }
+        result2 = {
+            'Digest': 'b'
+        }
+        mock_process = mock.Mock()
+        mock_process.communicate.side_effect = [
+            (json.dumps(result1), ''),
+            (json.dumps(result2), ''),
+        ]
+
+        mock_process.returncode = 0
+        mock_popen.return_value = mock_process
+
+        image = 'docker.io/tripleomaster/heat-docker-agents-centos'
+        tag = 'latest'
+        append_tag = 'modify-123'
+        push_destination = 'localhost:8787'
+        error = processutils.ProcessExecutionError(
+            '', 'ouch', -1, 'ansible-playbook')
+        mock_ansible.return_value.run.side_effect = error
+
+        self.assertRaises(
+            processutils.ProcessExecutionError,
+            self.uploader.upload_image,
+            image + ':' + tag, None, push_destination, set(), append_tag,
+            'add-foo-plugin', {'foo_version': '1.0.1'}
+        )
+
+        self.dockermock.assert_called_once_with(
+            base_url='unix://var/run/docker.sock', version='auto')
+
+        self.dockermock.return_value.pull.assert_called_once_with(
+            image, tag=tag, stream=True)
+        self.dockermock.return_value.tag.assert_not_called()
+        self.dockermock.return_value.push.assert_not_called()
+
     @mock.patch('requests.get')
     def test_is_insecure_registry_known(self, mock_get):
         self.assertFalse(

tripleo_common/tests/image/test_kolla_builder.py

@@ -669,7 +669,9 @@ class TestPrepare(base.TestCase):
                     'set': mapping_args,
                     'tag_from_label': 'bar',
                     'excludes': ['nova', 'neutron'],
-                    'push_destination': '192.0.2.1:8787'
+                    'push_destination': '192.0.2.1:8787',
+                    'modify_role': 'add-foo-plugin',
+                    'modify_vars': {'foo_version': '1.0.1'}
                 }]
             }
         }
@@ -709,7 +711,10 @@ class TestPrepare(base.TestCase):
                 pull_source=None,
                 push_destination=None,
                 service_filter=None,
-                tag_from_label='foo'
+                tag_from_label='foo',
+                append_tag=None,
+                modify_role=None,
+                modify_vars=None
             ),
             mock.call(
                 excludes=['nova', 'neutron'],
@@ -719,7 +724,10 @@ class TestPrepare(base.TestCase):
                 pull_source=None,
                 push_destination='192.0.2.1:8787',
                 service_filter=None,
-                tag_from_label='bar'
+                tag_from_label='bar',
+                append_tag=mock.ANY,
+                modify_role='add-foo-plugin',
+                modify_vars={'foo_version': '1.0.1'}
             )
         ])