From 90d15f65a63bacb233eb1d9d42345bd6e6e9e6d1 Mon Sep 17 00:00:00 2001 From: Abhishek Kekane Date: Wed, 14 May 2025 17:06:22 +0000 Subject: [PATCH] Add support for passing image size to Glance API Introduced a new command-line option --size for the image-create and image-create-via-import commands. Added the `x-openstack-image-size` header to transmit the image size from image-upload and image-stage commands to the Glance API. If --size is not specified but --file is provided, the script calculates the file size and includes it in the `x-openstack-image-size` header when invoking image-upload and image-stage commands. Change-Id: I7572f8a5d42a9968b940ed71eecbe3028e92877e Signed-off-by: Abhishek Kekane --- glanceclient/tests/unit/v2/base.py | 4 + glanceclient/tests/unit/v2/test_images.py | 71 +++++- glanceclient/tests/unit/v2/test_shell_v2.py | 261 +++++++++++++++++++- glanceclient/v2/images.py | 10 +- glanceclient/v2/shell.py | 43 +++- 5 files changed, 376 insertions(+), 13 deletions(-) diff --git a/glanceclient/tests/unit/v2/base.py b/glanceclient/tests/unit/v2/base.py index c595f41c..95f037b5 100644 --- a/glanceclient/tests/unit/v2/base.py +++ b/glanceclient/tests/unit/v2/base.py @@ -87,6 +87,10 @@ class BaseController(testtools.TestCase): resp = self.controller.upload(*args, **kwargs) self._assertRequestId(resp) + def stage(self, *args, **kwargs): + resp = self.controller.stage(*args, **kwargs) + self._assertRequestId(resp) + def data(self, *args, **kwargs): body = self.controller.data(*args, **kwargs) self._assertRequestId(body) diff --git a/glanceclient/tests/unit/v2/test_images.py b/glanceclient/tests/unit/v2/test_images.py index 8a82774b..67dfdd64 100644 --- a/glanceclient/tests/unit/v2/test_images.py +++ b/glanceclient/tests/unit/v2/test_images.py @@ -195,6 +195,12 @@ data_fixtures = { '', ), }, + '/v2/images/606b0e88-7c5a-4d54-b5bb-046105d4de6f/stage': { + 'PUT': ( + {}, + '', + ), + }, '/v2/images/5cc4bebc-db27-11e1-a1eb-080027cbe205/file': { 'GET': ( {}, @@ -1010,6 +1016,16 @@ class TestController(testtools.TestCase): self.assertEqual('3a4560a1-e585-443e-9b39-553b46ec92d1', image.id) self.assertEqual('image-1', image.name) + def test_create_image_w_size(self): + properties = { + 'name': 'image-1', + 'size': '4' + } + image = self.controller.create(**properties) + self.assertEqual('3a4560a1-e585-443e-9b39-553b46ec92d1', image.id) + self.assertEqual('image-1', image.name) + self.assertIsNone(image.get('size')) + def test_create_bad_additionalProperty_type(self): properties = { 'name': 'image-1', @@ -1054,12 +1070,63 @@ class TestController(testtools.TestCase): image_data)] self.assertEqual(expect, self.api.calls) - def test_data_upload_w_size(self): + def test_data_upload_with_invalid_size(self): + image_data = 'CCC' + image_id = '606b0e88-7c5a-4d54-b5bb-046105d4de6f' + self.assertRaises(TypeError, self.controller.upload, image_id, + image_data, image_size='invalid_size') + expect = [] + self.assertEqual(expect, self.api.calls) + + def test_data_upload_w_size_same_as_data(self): image_data = 'CCC' image_id = '606b0e88-7c5a-4d54-b5bb-046105d4de6f' self.controller.upload(image_id, image_data, image_size=3) expect = [('PUT', '/v2/images/%s/file' % image_id, - {'Content-Type': 'application/octet-stream'}, + {'Content-Type': 'application/octet-stream', + 'x-openstack-image-size': str(len(image_data))}, + image_data)] + self.assertEqual(expect, self.api.calls) + + def test_data_upload_w_size_diff_than_data(self): + image_data = 'CCCCCC' + image_size = '3' + image_id = '606b0e88-7c5a-4d54-b5bb-046105d4de6f' + self.controller.upload(image_id, image_data, + image_size=int(image_size)) + expect = [('PUT', '/v2/images/%s/file' % image_id, + {'Content-Type': 'application/octet-stream', + 'x-openstack-image-size': image_size}, + image_data)] + self.assertEqual(expect, self.api.calls) + + def test_data_stage_with_invalid_size(self): + image_data = 'CCC' + image_id = '606b0e88-7c5a-4d54-b5bb-046105d4de6f' + self.assertRaises(TypeError, self.controller.stage, image_id, + image_data, image_size='invalid_size') + expect = [] + self.assertEqual(expect, self.api.calls) + + def test_data_stage_w_size_same_as_data(self): + image_data = 'CCC' + image_id = '606b0e88-7c5a-4d54-b5bb-046105d4de6f' + self.controller.stage(image_id, image_data, image_size=3) + expect = [('PUT', '/v2/images/%s/stage' % image_id, + {'Content-Type': 'application/octet-stream', + 'x-openstack-image-size': str(len(image_data))}, + image_data)] + self.assertEqual(expect, self.api.calls) + + def test_data_stage_w_size_diff_than_data(self): + image_data = 'CCCCCC' + image_size = '3' + image_id = '606b0e88-7c5a-4d54-b5bb-046105d4de6f' + self.controller.stage(image_id, image_data, + image_size=int(image_size)) + expect = [('PUT', '/v2/images/%s/stage' % image_id, + {'Content-Type': 'application/octet-stream', + 'x-openstack-image-size': image_size}, image_data)] self.assertEqual(expect, self.api.calls) diff --git a/glanceclient/tests/unit/v2/test_shell_v2.py b/glanceclient/tests/unit/v2/test_shell_v2.py index e5a33b00..bdfb1396 100644 --- a/glanceclient/tests/unit/v2/test_shell_v2.py +++ b/glanceclient/tests/unit/v2/test_shell_v2.py @@ -679,6 +679,117 @@ class ShellV2Test(testtools.TestCase): {'quota': 'quota2', 'limit': 20, 'usage': 5}], ['Quota', 'Limit', 'Usage']) + def test_do_image_stage_size_match(self): + args = mock.Mock() + args.id = 'IMG-01' + args.file = 'testfile' + args.size = 1024 + args.progress = False + + with mock.patch('glanceclient.common.utils.get_data_file', + return_value='fileobj'), \ + mock.patch('glanceclient.common.utils.get_file_size', + return_value=1024), \ + mock.patch('glanceclient.v2.shell._validate_backend'), \ + mock.patch.object(self.gc.images, + 'stage') as mock_stage: + test_shell.do_image_stage(self.gc, args) + mock_stage.assert_called_once_with('IMG-01', 'fileobj', + 1024) + + def test_do_image_stage_size_mismatch(self): + args = mock.Mock() + args.id = 'IMG-01' + args.file = 'testfile' + args.size = 1024 + args.progress = False + + with mock.patch('glanceclient.common.utils.get_data_file', + return_value='fileobj'), \ + mock.patch('glanceclient.common.utils.get_file_size', + return_value=2048), \ + mock.patch('glanceclient.v2.shell._validate_backend'): + with self.assertRaisesRegex(ValueError, + "Size mismatch: provided size 1024 " + "does not match the size of the " + "image 2048"): + test_shell.do_image_stage(self.gc, args) + + def test_do_image_stage_no_size_in_args(self): + args = mock.Mock() + args.id = 'IMG-01' + args.file = 'testfile' + args.size = None + args.progress = False + + with mock.patch('glanceclient.common.utils.get_data_file', + return_value='fileobj'), \ + mock.patch('glanceclient.common.utils.get_file_size', + return_value=1024), \ + mock.patch('glanceclient.v2.shell._validate_backend'), \ + mock.patch.object(self.gc.images, + 'stage') as mock_upload: + test_shell.do_image_stage(self.gc, args) + mock_upload.assert_called_once_with('IMG-01', 'fileobj', + 1024) + + def test_do_image_upload_size_match(self): + args = mock.Mock() + args.id = 'IMG-01' + args.file = 'testfile' + args.size = 1024 + args.store = None + args.progress = False + + with mock.patch('glanceclient.common.utils.get_data_file', + return_value='fileobj'), \ + mock.patch('glanceclient.common.utils.get_file_size', + return_value=1024), \ + mock.patch('glanceclient.v2.shell._validate_backend'), \ + mock.patch.object(self.gc.images, + 'upload') as mock_upload: + test_shell.do_image_upload(self.gc, args) + mock_upload.assert_called_once_with('IMG-01', 'fileobj', + 1024, backend=None) + + def test_do_image_upload_size_mismatch(self): + args = mock.Mock() + args.id = 'IMG-01' + args.file = 'testfile' + args.size = 1024 + args.store = None + args.progress = False + + with mock.patch('glanceclient.common.utils.get_data_file', + return_value='fileobj'), \ + mock.patch('glanceclient.common.utils.get_file_size', + return_value=2048), \ + mock.patch('glanceclient.v2.shell._validate_backend'): + with self.assertRaisesRegex(ValueError, + "Size mismatch: provided size 1024 " + "does not match the size of the " + "image 2048"): + test_shell.do_image_upload(self.gc, args) + + def test_do_image_upload_no_size_in_args(self): + args = mock.Mock() + args.id = 'IMG-01' + args.file = 'testfile' + args.size = None + args.store = None + args.progress = False + + with mock.patch('glanceclient.common.utils.get_data_file', + return_value='fileobj'), \ + mock.patch('glanceclient.common.utils.get_file_size', + return_value=1024), \ + mock.patch('glanceclient.v2.shell._validate_backend'), \ + mock.patch.object(self.gc.images, + 'upload') as mock_upload: + test_shell.do_image_upload(self.gc, args) + mock_upload.assert_called_once_with('IMG-01', 'fileobj', + 1024, backend=None) + @mock.patch('sys.stdin', autospec=True) def test_do_image_create_no_user_props(self, mock_stdin): args = self._make_args({'name': 'IMG-01', 'disk_format': 'vhd', @@ -781,6 +892,77 @@ class ShellV2Test(testtools.TestCase): except Exception: pass + def _do_image_create(self, temp_args, expect_size=None): + self.mock_get_data_file.return_value = io.StringIO() + with open(tempfile.mktemp(), 'w+') as f: + f.write('Some data here') + f.flush() + f.seek(0) + file_name = f.name + self.addCleanup(lambda: os.remove(f.name) if os.path.exists( + f.name) else None) + temp_args = temp_args.copy() + temp_args['file'] = file_name + args = self._make_args(temp_args) + with mock.patch.object(self.gc.images, 'create') as mocked_create, \ + mock.patch.object(self.gc.images, 'get') as mocked_get, \ + mock.patch.object(utils, 'get_file_size') as mock_size: + expected_size = len('Some data here') + mock_size.return_value = expected_size + ignore_fields = ['self', 'access', 'schema'] + expect_image = dict( + [(field, field) for field in ignore_fields]) + expect_image['id'] = 'pass' + expect_image['name'] = 'IMG-01' + expect_image['disk_format'] = 'vhd' + expect_image['container_format'] = 'bare' + expect_image['checksum'] = 'fake-checksum' + expect_image['os_hash_algo'] = 'fake-hash_algo' + expect_image['os_hash_value'] = 'fake-hash_value' + if expect_size is not None: + expect_image['size'] = expect_size + mocked_create.return_value = expect_image + mocked_get.return_value = expect_image + + test_shell.do_image_create(self.gc, args) + + temp_args.pop('file', None) + mocked_create.assert_called_once_with(**temp_args) + mocked_get.assert_called_once_with('pass') + expected_dict = { + 'id': 'pass', 'name': 'IMG-01', + 'disk_format': 'vhd', + 'container_format': 'bare', + 'checksum': 'fake-checksum', + 'os_hash_algo': 'fake-hash_algo', + 'os_hash_value': 'fake-hash_value', + } + if expect_size is not None: + expected_dict['size'] = expect_size + utils.print_dict.assert_called_once_with(expected_dict) + mock_size.assert_called() + + def test_do_image_create_without_size(self): + self._do_image_create( + temp_args={'name': 'IMG-01', 'disk_format': 'vhd', + 'container_format': 'bare', 'progress': False}, + expect_size=14) + + def test_do_image_create_with_size_exits(self): + args = self._make_args({'name': 'test-image', 'size': 1234}) + expected_msg = ("Setting 'size' during image creation is " + "not supported. Please use --size only when " + "uploading data.") + with mock.patch('glanceclient.common.utils.exit') as mock_exit: + mock_exit.side_effect = self._mock_utils_exit + try: + test_shell.do_image_create(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + + mock_exit.assert_called_once_with(expected_msg) + @mock.patch('sys.stdin', autospec=True) def test_do_image_create_hidden_image(self, mock_stdin): args = self._make_args({'name': 'IMG-01', 'disk_format': 'vhd', @@ -1652,6 +1834,77 @@ class ShellV2Test(testtools.TestCase): 'id': 'via-stdin', 'name': 'Mortimer', 'disk_format': 'raw', 'container_format': 'bare'}) + def _image_create_via_import_with_file_helper( + self, with_access=True): + """Helper for image create via import with file tests.""" + @mock.patch('glanceclient.common.utils.get_file_size') + @mock.patch('glanceclient.v2.shell.do_image_import') + @mock.patch('os.access') + @mock.patch('sys.stdin', autospec=True) + def _test_with_access(mock_stdin, mock_access, + mock_do_import, mock_size): + mock_stdin.isatty = lambda: True + self.mock_get_data_file.return_value = io.StringIO() + mock_access.return_value = with_access + mock_size.return_value = 14 + with open(tempfile.mktemp(), 'w+') as f: + f.write('Some data here') + f.flush() + f.seek(0) + file_name = f.name + self.addCleanup( + lambda: os.remove(file_name) if os.path.exists( + file_name) else None) + my_args = self.base_args.copy() + my_args.update({'file': file_name}) + args = self._make_args(my_args) + with mock.patch.object( + self.gc.images, 'create') as mocked_create, \ + mock.patch.object( + self.gc.images, 'get') as mocked_get, \ + mock.patch.object( + self.gc.images, 'get_import_info') as mocked_info: + ignore_fields = ['self', 'access', 'schema'] + expect_image = dict( + [(field, field) for field in ignore_fields]) + expect_image['id'] = 'fake-image-id' + expect_image['name'] = 'Mortimer' + expect_image['disk_format'] = 'raw' + expect_image['container_format'] = 'bare' + mocked_create.return_value = expect_image + mocked_get.return_value = expect_image + mocked_info.return_value = self.import_info_response + + test_shell.do_image_create_via_import(self.gc, args) + mocked_create.assert_called_once() + mock_do_import.assert_called_once() + mocked_get.assert_called_with(expect_image['id']) + mock_size.assert_called_once() + utils.print_dict.assert_called_with({ + 'id': expect_image['id'], 'name': 'Mortimer', + 'disk_format': 'raw', 'container_format': 'bare' + }) + + _test_with_access() + + def test_image_create_via_import_with_file(self): + self._image_create_via_import_with_file_helper() + + def test_do_image_create_via_import_with_size_exits(self): + args = self._make_args({'name': 'test-image', 'size': 1234}) + expected_msg = ("Setting 'size' during image creation is not " + "supported. Please use --size only when " + "uploading data.") + with mock.patch('glanceclient.common.utils.exit') as mock_exit: + mock_exit.side_effect = self._mock_utils_exit + try: + test_shell.do_image_create(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + + mock_exit.assert_called_once_with(expected_msg) + @mock.patch('glanceclient.v2.shell.do_image_import') @mock.patch('glanceclient.v2.shell.do_image_stage') @mock.patch('os.access') @@ -2056,11 +2309,13 @@ class ShellV2Test(testtools.TestCase): {'id': 'IMG-01', 'file': 'test', 'size': 1024, 'progress': False}) with mock.patch.object(self.gc.images, 'upload') as mocked_upload: - utils.get_data_file = mock.Mock(return_value='testfile') + expected_data = '*' * 1024 + utils.get_data_file = mock.Mock(return_value=expected_data) + utils.get_file_size = mock.Mock(return_value=1024) mocked_upload.return_value = None test_shell.do_image_upload(self.gc, args) - mocked_upload.assert_called_once_with('IMG-01', 'testfile', 1024, - backend=None) + mocked_upload.assert_called_once_with('IMG-01', expected_data, + 1024, backend=None) @mock.patch('glanceclient.common.utils.exit') def test_image_upload_invalid_store(self, mock_utils_exit): diff --git a/glanceclient/v2/images.py b/glanceclient/v2/images.py index 94c3d12c..7fe68c47 100644 --- a/glanceclient/v2/images.py +++ b/glanceclient/v2/images.py @@ -295,12 +295,17 @@ class Controller(object): :param image_id: ID of the image to upload data for. :param image_data: File-like object supplying the data to upload. - :param image_size: Unused - present for backwards compatibility + :param image_size: If present pass it as header :param u_url: Upload url to upload the data to. :param backend: Backend store to upload image to. """ url = u_url or '/v2/images/%s/file' % image_id hdrs = {'Content-Type': 'application/octet-stream'} + if image_size is not None: + if not isinstance(image_size, int): + raise TypeError("image_size must be an integer, " + "got %s" % type(image_size).__name__) + hdrs.update({'x-openstack-image-size': '%i' % image_size}) if backend is not None: hdrs['x-image-meta-store'] = backend @@ -343,11 +348,12 @@ class Controller(object): :param image_id: ID of the image to upload data for. :param image_data: File-like object supplying the data to upload. - :param image_size: Unused - present for backwards compatibility + :param image_size: If present pass it to upload call """ url = '/v2/images/%s/stage' % image_id resp, body = self.upload(image_id, image_data, + image_size=image_size, u_url=url) return body, resp diff --git a/glanceclient/v2/shell.py b/glanceclient/v2/shell.py index 0b647876..e77bb346 100644 --- a/glanceclient/v2/shell.py +++ b/glanceclient/v2/shell.py @@ -49,6 +49,17 @@ def get_image_schema(): return IMAGE_SCHEMA +def get_filesize(args, image_data): + filesize = getattr(args, 'size', None) or utils.get_file_size(image_data) + if getattr(args, 'size', None) and getattr(args, 'file', None): + actual_size = utils.get_file_size(image_data) + if actual_size != args.size: + raise ValueError("Size mismatch: provided size %s does not match " + "the size of the image %s" % ( + args.size, actual_size)) + return filesize + + @utils.schema_args(get_image_schema, omit=['locations', 'os_hidden']) # NOTE(rosmaita): to make this option more intuitive for end users, we # do not use the Glance image property name 'os_hidden' here. This means @@ -66,6 +77,11 @@ def get_image_schema(): help=_('Local file that contains disk image to be uploaded ' 'during creation. Alternatively, the image data can be ' 'passed to the client via stdin.')) +@utils.arg('--size', metavar='', type=int, + help=_('Size in bytes of image to be uploaded. Default is to get ' + 'size from provided data object but this is supported in ' + 'case where size cannot be inferred.'), + default=None) @utils.arg('--progress', action='store_true', default=False, help=_('Show upload progress bar.')) @utils.arg('--store', metavar='', @@ -90,6 +106,11 @@ def do_image_create(gc, args): backend = args.store file_name = fields.pop('file', None) + if 'size' in fields: + utils.exit( + "Setting 'size' during image creation is not supported. " + "Please use --size only when uploading data.") + using_stdin = hasattr(sys.stdin, 'isatty') and not sys.stdin.isatty() if args.store and not (file_name or using_stdin): utils.exit("--store option should only be provided with --file " @@ -107,7 +128,6 @@ def do_image_create(gc, args): if utils.get_data_file(args) is not None: backend = fields.get('store', None) args.id = image['id'] - args.size = None do_image_upload(gc, args) image = gc.images.get(args.id) finally: @@ -128,6 +148,11 @@ def do_image_create(gc, args): help=_('Local file that contains disk image to be uploaded ' 'during creation. Alternatively, the image data can be ' 'passed to the client via stdin.')) +@utils.arg('--size', metavar='', type=int, + help=_('Size in bytes of image to be uploaded. Default is to get ' + 'size from provided data object but this is supported in ' + 'case where size cannot be inferred.'), + default=None) @utils.arg('--progress', action='store_true', default=False, help=_('Show upload progress bar.')) @utils.arg('--import-method', metavar='', @@ -205,6 +230,11 @@ def do_image_create_via_import(gc, args): fields[key] = value file_name = fields.pop('file', None) + if 'size' in fields: + utils.exit( + "Setting 'size' during image creation is not supported. " + "Please use --size only when uploading data.") + using_stdin = hasattr(sys.stdin, 'isatty') and not sys.stdin.isatty() # special processing for backward compatibility with image-create @@ -315,7 +345,6 @@ def do_image_create_via_import(gc, args): args.id = image['id'] if args.import_method: if utils.get_data_file(args) is not None: - args.size = None do_image_stage(gc, args) args.from_create = True args.stores = stores @@ -699,13 +728,14 @@ def do_image_upload(gc, args): _validate_backend(backend, gc) image_data = utils.get_data_file(args) + filesize = get_filesize(args, image_data) + if args.progress: - filesize = utils.get_file_size(image_data) if filesize is not None: # NOTE(kragniz): do not show a progress bar if the size of the # input is unknown (most likely a piped input) image_data = progressbar.VerboseFileWrapper(image_data, filesize) - gc.images.upload(args.id, image_data, args.size, backend=backend) + gc.images.upload(args.id, image_data, filesize, backend=backend) @utils.arg('--file', metavar='', @@ -724,13 +754,14 @@ def do_image_upload(gc, args): def do_image_stage(gc, args): """Upload data for a specific image to staging.""" image_data = utils.get_data_file(args) + filesize = get_filesize(args, image_data) + if args.progress: - filesize = utils.get_file_size(image_data) if filesize is not None: # NOTE(kragniz): do not show a progress bar if the size of the # input is unknown (most likely a piped input) image_data = progressbar.VerboseFileWrapper(image_data, filesize) - gc.images.stage(args.id, image_data, args.size) + gc.images.stage(args.id, image_data, filesize) @utils.arg('--import-method', metavar='', default='glance-direct',