From 7aeba8f51afbd2593886318af59697c83932b378 Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Thu, 20 Feb 2020 22:41:18 -0500 Subject: [PATCH] Add --work-dir option to container image build command Add --work-dir to openstack overcloud container image build command and every run will create a unique workspace which where will be stored Kolla configs and build logs. Default directory will be in /tmp/container-builds. UUIDs are used to identify each time we run the command and will be the directory name in the work dir. Depends-On: https://review.opendev.org/#/c/709389/ Related-Bug: #1864108 Change-Id: Id3e52ba920c54c98529ecb5f723ba452362a0b32 --- .../container_work_dir-edb40007cb25168e.yaml | 8 ++ .../tests/v1/test_container_image.py | 52 ++++++---- tripleoclient/v1/container_image.py | 97 +++++++++++-------- 3 files changed, 97 insertions(+), 60 deletions(-) create mode 100644 releasenotes/notes/container_work_dir-edb40007cb25168e.yaml diff --git a/releasenotes/notes/container_work_dir-edb40007cb25168e.yaml b/releasenotes/notes/container_work_dir-edb40007cb25168e.yaml new file mode 100644 index 000000000..64602d14a --- /dev/null +++ b/releasenotes/notes/container_work_dir-edb40007cb25168e.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Add --work-dir to openstack overcloud container image build command and + every run will create a unique workspace which where will be stored Kolla + configs and build logs. Default directory will be in + /tmp/container-builds. UUIDs are used to identify each time we run the + command and will be the directory name in the work dir. diff --git a/tripleoclient/tests/v1/test_container_image.py b/tripleoclient/tests/v1/test_container_image.py index 593874257..cc8f54e62 100644 --- a/tripleoclient/tests/v1/test_container_image.py +++ b/tripleoclient/tests/v1/test_container_image.py @@ -21,6 +21,7 @@ import shutil import six import sys import tempfile +import uuid import yaml from osc_lib import exceptions as oscexc @@ -927,7 +928,11 @@ class TestContainerImageBuild(TestPluginV1): # Get the command object to test self.cmd = container_image.BuildImage(self.app, None) self.cmd.app.stdout = six.StringIO() - self.temp_dir = self.useFixture(fixtures.TempDir()).join() + self.uuid = str(uuid.uuid4()) + self.temp_dir = os.path.join(self.useFixture( + fixtures.TempDir()).join()) + self.temp_dir_uuid = os.path.join(self.temp_dir, self.uuid) + # Default conf file self.default_kolla_conf = os.path.join( sys.prefix, 'share', 'tripleo-common', 'container-images', @@ -936,7 +941,9 @@ class TestContainerImageBuild(TestPluginV1): @mock.patch('sys.exit') @mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder', autospec=True) - def test_container_image_build_noargs(self, mock_builder, exit_mock): + @mock.patch('os.makedirs') + def test_container_image_build_noargs(self, mock_mkdirs, mock_builder, + exit_mock): arglist = [] verifylist = [] mock_builder.return_value.build_images.return_value = 'done' @@ -953,38 +960,39 @@ class TestContainerImageBuild(TestPluginV1): @mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder', autospec=True) - def test_container_image_build(self, mock_builder): + @mock.patch('uuid.uuid4') + def test_container_image_build(self, mock_uuid, mock_builder): arglist = [ '--config-file', '/tmp/foo.yaml', '--config-file', '/tmp/bar.yaml', + '--work-dir', + self.temp_dir, '--kolla-config-file', '/tmp/kolla.conf' ] verifylist = [] mock_builder.return_value.build_images.return_value = 'done' + mock_uuid.return_value = self.uuid parsed_args = self.check_parser(self.cmd, arglist, verifylist) f, path = tempfile.mkstemp(dir=self.temp_dir) - with mock.patch('tempfile.mkdtemp') as mock_mkd: - mock_mkd.return_value = '/tmp/testing' - with mock.patch('tempfile.mkstemp') as mock_mkstemp: - with mock.patch('os.chdir'): - mock_mkstemp.return_value = f, path - self.cmd.take_action(parsed_args) + with mock.patch('tempfile.mkstemp') as mock_mkstemp: + with mock.patch('os.chdir'): + mock_mkstemp.return_value = f, path + self.cmd.take_action(parsed_args) mock_builder.assert_called_once_with([ '/tmp/foo.yaml', '/tmp/bar.yaml']) mock_builder.return_value.build_images.assert_called_once_with([ self.default_kolla_conf, '/tmp/kolla.conf', path - ], [], False, '/tmp/testing') + ], [], False, self.temp_dir_uuid) @mock.patch('os.chdir') @mock.patch('os.fdopen', autospec=True) - @mock.patch('tempfile.mkdtemp') @mock.patch('tempfile.mkstemp') @mock.patch( 'tripleoclient.utils.get_from_cfg') @@ -1001,14 +1009,15 @@ class TestContainerImageBuild(TestPluginV1): mock_builder, mock_buildah, mock_kolla_boolean_cfg, mock_kolla_cfg, mock_mkstemp, - mock_mkdtemp, mock_fdopen, - mock_chdir): + mock_fdopen, mock_chdir): arglist = [ '--config-file', '/tmp/bar.yaml', '--kolla-config-file', '/tmp/kolla.conf', - '--use-buildah' + '--use-buildah', + '--work-dir', + self.temp_dir, ] parsed_args = self.check_parser(self.cmd, arglist, []) @@ -1016,7 +1025,6 @@ class TestContainerImageBuild(TestPluginV1): mock_builder.return_value.build_images.return_value = deps mock_mkstemp.return_value = (1, '/tmp/whatever_file') - mock_mkdtemp.return_value = '/tmp/whatever_dir' mock_bb = mock.MagicMock() mock_bb.build_all = mock.MagicMock() @@ -1042,7 +1050,8 @@ class TestContainerImageBuild(TestPluginV1): @mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder', autospec=True) - def test_container_image_build_with_exclude(self, mock_builder): + @mock.patch('uuid.uuid4') + def test_container_image_build_with_exclude(self, mock_uuid, mock_builder): arglist = [ '--config-file', '/tmp/foo.yaml', @@ -1050,6 +1059,8 @@ class TestContainerImageBuild(TestPluginV1): '/tmp/bar.yaml', '--kolla-config-file', '/tmp/kolla.conf', + '--work-dir', + '/tmp/testing', '--exclude', 'foo', '--exclude', @@ -1057,6 +1068,7 @@ class TestContainerImageBuild(TestPluginV1): ] verifylist = [] mock_builder.return_value.build_images.return_value = 'done' + mock_uuid.return_value = '123' parsed_args = self.check_parser(self.cmd, arglist, verifylist) @@ -1073,7 +1085,7 @@ class TestContainerImageBuild(TestPluginV1): mock_builder.return_value.build_images.assert_called_once_with([ self.default_kolla_conf, '/tmp/kolla.conf', path - ], ['foo', 'bar'], False, '/tmp/testing') + ], ['foo', 'bar'], False, '/tmp/testing/123') @mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder', autospec=True) @@ -1084,6 +1096,8 @@ class TestContainerImageBuild(TestPluginV1): '--list-images', '--config-file', '/tmp/bar.yaml', + '--work-dir', + '/tmp/testing', '--kolla-config-file', '/tmp/kolla.conf' ] @@ -1105,7 +1119,9 @@ class TestContainerImageBuild(TestPluginV1): @mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder', autospec=True) @mock.patch('os.remove') - def test_container_image_build_list_deps(self, mock_remove, mock_builder): + @mock.patch('os.makedirs') + def test_container_image_build_list_deps(self, mock_mkdirs, mock_remove, + mock_builder): arglist = [ '--config-file', '/tmp/bar.yaml', diff --git a/tripleoclient/v1/container_image.py b/tripleoclient/v1/container_image.py index a69c0d771..c5f506545 100644 --- a/tripleoclient/v1/container_image.py +++ b/tripleoclient/v1/container_image.py @@ -22,6 +22,7 @@ import shutil import sys import tempfile import time +import uuid from osc_lib import exceptions as oscexc from osc_lib.i18n import _ @@ -182,6 +183,14 @@ class BuildImage(command.Command): help=_('Use Buildah instead of Docker to build the images ' 'with Kolla.') ) + parser.add_argument( + "--work-dir", + dest="work_dir", + default='/tmp/container-builds', + metavar='', + help=_("TripleO container builds directory, storing configs and " + "logs for each image and its dependencies.") + ) return parser def take_action(self, parsed_args): @@ -194,49 +203,53 @@ class BuildImage(command.Command): tmp.write('list_dependencies=true') kolla_config_files = list(parsed_args.kolla_config_files) kolla_config_files.append(path) - with utils.TempDirs(dir_prefix='kolla-') as kolla_tmp_dir: - try: - builder = kolla_builder.KollaImageBuilder( - parsed_args.config_files - ) - result = builder.build_images(kolla_config_files, - parsed_args.excludes, - parsed_args.use_buildah, - kolla_tmp_dir) + # Generate an unique work directory so we can keep configs and logs + # each time we run the command; they'll be stored in work_dir. + kolla_work_dir = os.path.join(parsed_args.work_dir, str(uuid.uuid4())) - if parsed_args.use_buildah: - deps = json.loads(result) - kolla_cfg = utils.get_read_config(kolla_config_files) - bb = buildah.BuildahBuilder( - kolla_tmp_dir, deps, - utils.get_from_cfg(kolla_cfg, "base"), - utils.get_from_cfg(kolla_cfg, "type"), - utils.get_from_cfg(kolla_cfg, "tag"), - utils.get_from_cfg(kolla_cfg, "namespace"), - utils.get_from_cfg(kolla_cfg, "registry"), - utils.getboolean_from_cfg(kolla_cfg, "push")) - bb.build_all() - elif parsed_args.list_dependencies: - deps = json.loads(result) - yaml.safe_dump( - deps, - self.app.stdout, - indent=2, - default_flow_style=False - ) - elif parsed_args.list_images: - deps = json.loads(result) - images = [] - BuildImage.images_from_deps(images, deps) - yaml.safe_dump( - images, - self.app.stdout, - default_flow_style=False - ) - elif result: - self.app.stdout.write(result) - finally: - os.remove(path) + # Make sure the unique work directory exists + if not os.path.exists(kolla_work_dir): + self.log.debug("Creating container builds " + "workspace in: %s" % kolla_work_dir) + os.makedirs(kolla_work_dir) + + builder = kolla_builder.KollaImageBuilder(parsed_args.config_files) + result = builder.build_images(kolla_config_files, + parsed_args.excludes, + parsed_args.use_buildah, + kolla_work_dir) + + if parsed_args.use_buildah: + deps = json.loads(result) + kolla_cfg = utils.get_read_config(kolla_config_files) + bb = buildah.BuildahBuilder( + kolla_work_dir, deps, + utils.get_from_cfg(kolla_cfg, "base"), + utils.get_from_cfg(kolla_cfg, "type"), + utils.get_from_cfg(kolla_cfg, "tag"), + utils.get_from_cfg(kolla_cfg, "namespace"), + utils.get_from_cfg(kolla_cfg, "registry"), + utils.getboolean_from_cfg(kolla_cfg, "push")) + bb.build_all() + elif parsed_args.list_dependencies: + deps = json.loads(result) + yaml.safe_dump( + deps, + self.app.stdout, + indent=2, + default_flow_style=False + ) + elif parsed_args.list_images: + deps = json.loads(result) + images = [] + BuildImage.images_from_deps(images, deps) + yaml.safe_dump( + images, + self.app.stdout, + default_flow_style=False + ) + elif result: + self.app.stdout.write(result) class PrepareImageFiles(command.Command):