Add --work-dir option to container image build command

Add --work-dir to openstack overcloud container image build command and
every run will create a unique workspace which where will be stored Kolla
configs and build logs. Default directory will be in
/tmp/container-builds. UUIDs are used to identify each time we run the
command and will be the directory name in the work dir.

Depends-On: https://review.opendev.org/#/c/709389/
Related-Bug: #1864108
Change-Id: Id3e52ba920c54c98529ecb5f723ba452362a0b32

releasenotes/notes/container_work_dir-edb40007cb25168e.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Add --work-dir to openstack overcloud container image build command and
+    every run will create a unique workspace which where will be stored Kolla
+    configs and build logs. Default directory will be in
+    /tmp/container-builds. UUIDs are used to identify each time we run the
+    command and will be the directory name in the work dir.

tripleoclient/tests/v1/test_container_image.py

@@ -21,6 +21,7 @@ import shutil
 import six
 import sys
 import tempfile
+import uuid
 import yaml
 
 from osc_lib import exceptions as oscexc
@@ -927,7 +928,11 @@ class TestContainerImageBuild(TestPluginV1):
         # Get the command object to test
         self.cmd = container_image.BuildImage(self.app, None)
         self.cmd.app.stdout = six.StringIO()
-        self.temp_dir = self.useFixture(fixtures.TempDir()).join()
+        self.uuid = str(uuid.uuid4())
+        self.temp_dir = os.path.join(self.useFixture(
+                                     fixtures.TempDir()).join())
+        self.temp_dir_uuid = os.path.join(self.temp_dir, self.uuid)
+
         # Default conf file
         self.default_kolla_conf = os.path.join(
             sys.prefix, 'share', 'tripleo-common', 'container-images',
@@ -936,7 +941,9 @@ class TestContainerImageBuild(TestPluginV1):
     @mock.patch('sys.exit')
     @mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder',
                 autospec=True)
-    def test_container_image_build_noargs(self, mock_builder, exit_mock):
+    @mock.patch('os.makedirs')
+    def test_container_image_build_noargs(self, mock_mkdirs, mock_builder,
+                                          exit_mock):
         arglist = []
         verifylist = []
         mock_builder.return_value.build_images.return_value = 'done'
@@ -953,38 +960,39 @@ class TestContainerImageBuild(TestPluginV1):
 
     @mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder',
                 autospec=True)
-    def test_container_image_build(self, mock_builder):
+    @mock.patch('uuid.uuid4')
+    def test_container_image_build(self, mock_uuid, mock_builder):
         arglist = [
             '--config-file',
             '/tmp/foo.yaml',
             '--config-file',
             '/tmp/bar.yaml',
+            '--work-dir',
+            self.temp_dir,
             '--kolla-config-file',
             '/tmp/kolla.conf'
         ]
         verifylist = []
         mock_builder.return_value.build_images.return_value = 'done'
+        mock_uuid.return_value = self.uuid
 
         parsed_args = self.check_parser(self.cmd, arglist, verifylist)
 
         f, path = tempfile.mkstemp(dir=self.temp_dir)
-        with mock.patch('tempfile.mkdtemp') as mock_mkd:
+        with mock.patch('tempfile.mkstemp') as mock_mkstemp:
-            mock_mkd.return_value = '/tmp/testing'
+            with mock.patch('os.chdir'):
-            with mock.patch('tempfile.mkstemp') as mock_mkstemp:
+                mock_mkstemp.return_value = f, path
-                with mock.patch('os.chdir'):
+                self.cmd.take_action(parsed_args)
-                    mock_mkstemp.return_value = f, path
-                    self.cmd.take_action(parsed_args)
 
         mock_builder.assert_called_once_with([
             '/tmp/foo.yaml', '/tmp/bar.yaml'])
         mock_builder.return_value.build_images.assert_called_once_with([
             self.default_kolla_conf, '/tmp/kolla.conf',
             path
-        ], [], False, '/tmp/testing')
+        ], [], False, self.temp_dir_uuid)
 
     @mock.patch('os.chdir')
     @mock.patch('os.fdopen', autospec=True)
-    @mock.patch('tempfile.mkdtemp')
     @mock.patch('tempfile.mkstemp')
     @mock.patch(
         'tripleoclient.utils.get_from_cfg')
@@ -1001,14 +1009,15 @@ class TestContainerImageBuild(TestPluginV1):
                                                 mock_builder, mock_buildah,
                                                 mock_kolla_boolean_cfg,
                                                 mock_kolla_cfg, mock_mkstemp,
-                                                mock_mkdtemp, mock_fdopen,
+                                                mock_fdopen, mock_chdir):
-                                                mock_chdir):
         arglist = [
             '--config-file',
             '/tmp/bar.yaml',
             '--kolla-config-file',
             '/tmp/kolla.conf',
-            '--use-buildah'
+            '--use-buildah',
+            '--work-dir',
+            self.temp_dir,
         ]
 
         parsed_args = self.check_parser(self.cmd, arglist, [])
@@ -1016,7 +1025,6 @@ class TestContainerImageBuild(TestPluginV1):
         mock_builder.return_value.build_images.return_value = deps
 
         mock_mkstemp.return_value = (1, '/tmp/whatever_file')
-        mock_mkdtemp.return_value = '/tmp/whatever_dir'
 
         mock_bb = mock.MagicMock()
         mock_bb.build_all = mock.MagicMock()
@@ -1042,7 +1050,8 @@ class TestContainerImageBuild(TestPluginV1):
 
     @mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder',
                 autospec=True)
-    def test_container_image_build_with_exclude(self, mock_builder):
+    @mock.patch('uuid.uuid4')
+    def test_container_image_build_with_exclude(self, mock_uuid, mock_builder):
         arglist = [
             '--config-file',
             '/tmp/foo.yaml',
@@ -1050,6 +1059,8 @@ class TestContainerImageBuild(TestPluginV1):
             '/tmp/bar.yaml',
             '--kolla-config-file',
             '/tmp/kolla.conf',
+            '--work-dir',
+            '/tmp/testing',
             '--exclude',
             'foo',
             '--exclude',
@@ -1057,6 +1068,7 @@ class TestContainerImageBuild(TestPluginV1):
         ]
         verifylist = []
         mock_builder.return_value.build_images.return_value = 'done'
+        mock_uuid.return_value = '123'
 
         parsed_args = self.check_parser(self.cmd, arglist, verifylist)
 
@@ -1073,7 +1085,7 @@ class TestContainerImageBuild(TestPluginV1):
         mock_builder.return_value.build_images.assert_called_once_with([
             self.default_kolla_conf, '/tmp/kolla.conf',
             path
-        ], ['foo', 'bar'], False, '/tmp/testing')
+        ], ['foo', 'bar'], False, '/tmp/testing/123')
 
     @mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder',
                 autospec=True)
@@ -1084,6 +1096,8 @@ class TestContainerImageBuild(TestPluginV1):
             '--list-images',
             '--config-file',
             '/tmp/bar.yaml',
+            '--work-dir',
+            '/tmp/testing',
             '--kolla-config-file',
             '/tmp/kolla.conf'
         ]
@@ -1105,7 +1119,9 @@ class TestContainerImageBuild(TestPluginV1):
     @mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder',
                 autospec=True)
     @mock.patch('os.remove')
-    def test_container_image_build_list_deps(self, mock_remove, mock_builder):
+    @mock.patch('os.makedirs')
+    def test_container_image_build_list_deps(self, mock_mkdirs, mock_remove,
+                                             mock_builder):
         arglist = [
             '--config-file',
             '/tmp/bar.yaml',

tripleoclient/v1/container_image.py

@@ -22,6 +22,7 @@ import shutil
 import sys
 import tempfile
 import time
+import uuid
 
 from osc_lib import exceptions as oscexc
 from osc_lib.i18n import _
@@ -182,6 +183,14 @@ class BuildImage(command.Command):
             help=_('Use Buildah instead of Docker to build the images '
                    'with Kolla.')
         )
+        parser.add_argument(
+            "--work-dir",
+            dest="work_dir",
+            default='/tmp/container-builds',
+            metavar='<container builds directory>',
+            help=_("TripleO container builds directory, storing configs and "
+                   "logs for each image and its dependencies.")
+        )
         return parser
 
     def take_action(self, parsed_args):
@@ -194,49 +203,53 @@ class BuildImage(command.Command):
                 tmp.write('list_dependencies=true')
         kolla_config_files = list(parsed_args.kolla_config_files)
         kolla_config_files.append(path)
-        with utils.TempDirs(dir_prefix='kolla-') as kolla_tmp_dir:
+        # Generate an unique work directory so we can keep configs and logs
-            try:
+        # each time we run the command; they'll be stored in work_dir.
-                builder = kolla_builder.KollaImageBuilder(
+        kolla_work_dir = os.path.join(parsed_args.work_dir, str(uuid.uuid4()))
-                    parsed_args.config_files
-                )
-                result = builder.build_images(kolla_config_files,
-                                              parsed_args.excludes,
-                                              parsed_args.use_buildah,
-                                              kolla_tmp_dir)
 
-                if parsed_args.use_buildah:
+        # Make sure the unique work directory exists
-                    deps = json.loads(result)
+        if not os.path.exists(kolla_work_dir):
-                    kolla_cfg = utils.get_read_config(kolla_config_files)
+            self.log.debug("Creating container builds "
-                    bb = buildah.BuildahBuilder(
+                           "workspace in: %s" % kolla_work_dir)
-                        kolla_tmp_dir, deps,
+            os.makedirs(kolla_work_dir)
-                        utils.get_from_cfg(kolla_cfg, "base"),
+
-                        utils.get_from_cfg(kolla_cfg, "type"),
+        builder = kolla_builder.KollaImageBuilder(parsed_args.config_files)
-                        utils.get_from_cfg(kolla_cfg, "tag"),
+        result = builder.build_images(kolla_config_files,
-                        utils.get_from_cfg(kolla_cfg, "namespace"),
+                                      parsed_args.excludes,
-                        utils.get_from_cfg(kolla_cfg, "registry"),
+                                      parsed_args.use_buildah,
-                        utils.getboolean_from_cfg(kolla_cfg, "push"))
+                                      kolla_work_dir)
-                    bb.build_all()
+
-                elif parsed_args.list_dependencies:
+        if parsed_args.use_buildah:
-                    deps = json.loads(result)
+            deps = json.loads(result)
-                    yaml.safe_dump(
+            kolla_cfg = utils.get_read_config(kolla_config_files)
-                        deps,
+            bb = buildah.BuildahBuilder(
-                        self.app.stdout,
+                kolla_work_dir, deps,
-                        indent=2,
+                utils.get_from_cfg(kolla_cfg, "base"),
-                        default_flow_style=False
+                utils.get_from_cfg(kolla_cfg, "type"),
-                    )
+                utils.get_from_cfg(kolla_cfg, "tag"),
-                elif parsed_args.list_images:
+                utils.get_from_cfg(kolla_cfg, "namespace"),
-                    deps = json.loads(result)
+                utils.get_from_cfg(kolla_cfg, "registry"),
-                    images = []
+                utils.getboolean_from_cfg(kolla_cfg, "push"))
-                    BuildImage.images_from_deps(images, deps)
+            bb.build_all()
-                    yaml.safe_dump(
+        elif parsed_args.list_dependencies:
-                        images,
+            deps = json.loads(result)
-                        self.app.stdout,
+            yaml.safe_dump(
-                        default_flow_style=False
+                deps,
-                    )
+                self.app.stdout,
-                elif result:
+                indent=2,
-                    self.app.stdout.write(result)
+                default_flow_style=False
-            finally:
+            )
-                os.remove(path)
+        elif parsed_args.list_images:
+            deps = json.loads(result)
+            images = []
+            BuildImage.images_from_deps(images, deps)
+            yaml.safe_dump(
+                images,
+                self.app.stdout,
+                default_flow_style=False
+            )
+        elif result:
+            self.app.stdout.write(result)
 
 
 class PrepareImageFiles(command.Command):