Add different upload cleanup behaviours

By default, images left over from upload operations are deleted from
the local docker, however there are cases where this is not desirable:
- when installing an undercloud, deleting the images then
  re-downloading from the undercloud repository wastes time
- when developing with a workflow where you do not want local images
  to be deleted ever (to save time, or preserve local image changes)

Change-Id: Id844aa2fa5ee20ad264b2fada75fa6cffdc1e307
Blueprint: container-prepare-workflow

tripleo_common/image/image_uploader.py

@@ -48,6 +48,12 @@ SECURE_REGISTRIES = (
     'docker.io',
 )
 
+CLEANUP = (
+    CLEANUP_FULL, CLEANUP_PARTIAL, CLEANUP_NONE
+) = (
+    'full', 'partial', 'none'
+)
+
 
 def get_undercloud_registry():
     addr = 'localhost'
@@ -66,12 +72,13 @@ class ImageUploadManager(BaseImageManager):
        """
 
     def __init__(self, config_files=None, verbose=False, debug=False,
-                 dry_run=False):
+                 dry_run=False, cleanup=CLEANUP_FULL):
         if config_files is None:
             config_files = []
         super(ImageUploadManager, self).__init__(config_files)
         self.uploaders = {}
         self.dry_run = dry_run
+        self.cleanup = cleanup
 
     def discover_image_tag(self, image, tag_from_label=None):
         uploader = self.uploader('docker')
@@ -108,7 +115,8 @@ class ImageUploadManager(BaseImageManager):
 
             self.uploader(uploader).add_upload_task(
                 image_name, pull_source, push_destination,
-                append_tag, modify_role, modify_vars, self.dry_run)
+                append_tag, modify_role, modify_vars, self.dry_run,
+                self.cleanup)
 
         for uploader in self.uploaders.values():
             uploader.run_tasks()
@@ -133,7 +141,8 @@ class ImageUploader(object):
 
     @abc.abstractmethod
     def add_upload_task(self, image_name, pull_source, push_destination,
-                        append_tag, modify_role, modify_vars, dry_run):
+                        append_tag, modify_role, modify_vars, dry_run,
+                        cleanup):
         """Add an upload task to be executed later"""
         pass
 
@@ -199,7 +208,7 @@ class DockerImageUploader(ImageUploader):
     @staticmethod
     def upload_image(image_name, pull_source, push_destination,
                      insecure_registries, append_tag, modify_role,
-                     modify_vars, dry_run):
+                     modify_vars, dry_run, cleanup):
         LOG.info('imagename: %s' % image_name)
         if ':' in image_name:
             image = image_name.rpartition(':')[0]
@@ -254,7 +263,11 @@ class DockerImageUploader(ImageUploader):
         DockerImageUploader._push(dockerc, target_image_no_tag, tag=target_tag)
 
         LOG.info('Completed upload for image %s' % image_name)
-        return source_image, target_image
+        if cleanup == CLEANUP_NONE:
+            return []
+        if cleanup == CLEANUP_PARTIAL:
+            return [source_image]
+        return [source_image, target_image]
 
     @staticmethod
     @tenacity.retry(  # Retry up to 5 times with jittered exponential backoff
@@ -436,7 +449,8 @@ class DockerImageUploader(ImageUploader):
                     LOG.warning(e)
 
     def add_upload_task(self, image_name, pull_source, push_destination,
-                        append_tag, modify_role, modify_vars, dry_run):
+                        append_tag, modify_role, modify_vars, dry_run,
+                        cleanup):
         # prime self.insecure_registries
         if pull_source:
             self.is_insecure_registry(self._image_to_url(pull_source).netloc)
@@ -445,7 +459,7 @@ class DockerImageUploader(ImageUploader):
         self.is_insecure_registry(self._image_to_url(push_destination).netloc)
         self.upload_tasks.append((image_name, pull_source, push_destination,
                                   self.insecure_registries, append_tag,
-                                  modify_role, modify_vars, dry_run))
+                                  modify_role, modify_vars, dry_run, cleanup))
 
     def run_tasks(self):
         if not self.upload_tasks:

tripleo_common/image/kolla_builder.py

@@ -98,7 +98,8 @@ def build_service_filter(environment, roles_data):
     return containerized_services.intersection(enabled_services)
 
 
-def container_images_prepare_multi(environment, roles_data, dry_run=False):
+def container_images_prepare_multi(environment, roles_data, dry_run=False,
+                                   cleanup=image_uploader.CLEANUP_FULL):
     """Perform multiple container image prepares and merge result
 
     Given the full heat environment and roles data, perform multiple image
@@ -160,6 +161,7 @@ def container_images_prepare_multi(environment, roles_data, dry_run=False):
                     [f.name],
                     verbose=True,
                     dry_run=dry_run,
+                    cleanup=cleanup
                 )
                 uploader.upload()
     return env_params

tripleo_common/tests/image/test_image_uploader.py

@@ -162,14 +162,21 @@ class TestDockerImageUploader(base.TestCase):
         push_destination = 'localhost:8787'
         push_image = 'localhost:8787/tripleomaster/heat-docker-agents-centos'
 
-        self.uploader.upload_image(image + ':' + tag,
+        self.assertEqual(
-                                   None,
+            ['docker.io/tripleomaster/heat-docker-agents-centos:latest',
-                                   push_destination,
+             'localhost:8787/tripleomaster/heat-docker-agents-centos:latest'],
-                                   set(),
+            self.uploader.upload_image(
-                                   None,
+                image + ':' + tag,
-                                   None,
+                None,
-                                   None,
+                push_destination,
-                                   False)
+                set(),
+                None,
+                None,
+                None,
+                False,
+                'full'
+            )
+        )
 
         self.dockermock.assert_called_once_with(
             base_url='unix://var/run/docker.sock', version='auto')
@@ -198,7 +205,8 @@ class TestDockerImageUploader(base.TestCase):
                                    None,
                                    None,
                                    None,
-                                   False)
+                                   False,
+                                   'full')
 
         self.dockermock.assert_called_once_with(
             base_url='unix://var/run/docker.sock', version='auto')
@@ -226,14 +234,20 @@ class TestDockerImageUploader(base.TestCase):
         tag = 'latest'
         push_destination = 'localhost:8787'
 
-        self.uploader.upload_image(image + ':' + tag,
+        self.assertEqual(
-                                   None,
+            [],
-                                   push_destination,
+            self.uploader.upload_image(
-                                   set(),
+                image + ':' + tag,
-                                   None,
+                None,
-                                   None,
+                push_destination,
-                                   None,
+                set(),
-                                   False)
+                None,
+                None,
+                None,
+                False,
+                'full'
+            )
+        )
 
         # both digests are the same, no pull/push
         self.dockermock.assert_not_called()
@@ -275,14 +289,21 @@ class TestDockerImageUploader(base.TestCase):
             'hosts': 'localhost'
         }]
 
-        self.uploader.upload_image(image + ':' + tag,
+        # test response for a partial cleanup
-                                   None,
+        self.assertEqual(
-                                   push_destination,
+            ['docker.io/tripleomaster/heat-docker-agents-centos:latest'],
-                                   set(),
+            self.uploader.upload_image(
-                                   append_tag,
+                image + ':' + tag,
-                                   'add-foo-plugin',
+                None,
-                                   {'foo_version': '1.0.1'},
+                push_destination,
-                                   False)
+                set(),
+                append_tag,
+                'add-foo-plugin',
+                {'foo_version': '1.0.1'},
+                False,
+                'partial'
+            )
+        )
 
         self.dockermock.assert_called_once_with(
             base_url='unix://var/run/docker.sock', version='auto')
@@ -322,7 +343,7 @@ class TestDockerImageUploader(base.TestCase):
             processutils.ProcessExecutionError,
             self.uploader.upload_image,
             image + ':' + tag, None, push_destination, set(), append_tag,
-            'add-foo-plugin', {'foo_version': '1.0.1'}, False
+            'add-foo-plugin', {'foo_version': '1.0.1'}, False, 'full'
         )
 
         self.dockermock.assert_called_once_with(
@@ -353,7 +374,8 @@ class TestDockerImageUploader(base.TestCase):
             append_tag,
             'add-foo-plugin',
             {'foo_version': '1.0.1'},
-            True
+            True,
+            'full'
         )
 
         self.dockermock.assert_not_called()
@@ -381,7 +403,8 @@ class TestDockerImageUploader(base.TestCase):
             append_tag,
             'add-foo-plugin',
             {'foo_version': '1.0.1'},
-            False
+            False,
+            'full'
         )
 
         self.dockermock.assert_not_called()

tripleo_common/tests/image/test_kolla_builder.py

@@ -911,7 +911,8 @@ class TestPrepare(base.TestCase):
             )
         ])
 
-        mock_im.assert_called_once_with(mock.ANY, dry_run=True, verbose=True)
+        mock_im.assert_called_once_with(mock.ANY, dry_run=True, verbose=True,
+                                        cleanup='full')
 
         self.assertEqual(
             {