From 80dc87cd8997c32a4a414cae2148f8ff5af62458 Mon Sep 17 00:00:00 2001 From: Jorge Merlino Date: Fri, 23 Dec 2022 09:46:43 -0300 Subject: [PATCH] Increase size of volume image metadata values Volume image metadata values were limited to 255 characters but Glance allows up to 65535 (considering it uses a TEXT field in MySQL). Cinder database also uses a TEXT field for those values so it made no sense to limit them to 255. The actual values could already be longer when they were copied from the image at volume creation time. Closes-Bug: #1988942 Change-Id: Id200ae93384a452b34bdd20dd1f3fc656ec5650f (cherry picked from commit 0bd1bd699d51162557dd0791ac6e79cb3149db8c) (cherry picked from commit 2fb2ff99b4cbf615163dbd663b853da590cd8995) --- cinder/api/schemas/volume_image_metadata.py | 2 +- cinder/api/validation/parameter_types.py | 9 +++ cinder/api/validation/validators.py | 8 +++ .../api/contrib/test_volume_image_metadata.py | 65 ++++++++++++++++++- ...tadata-size-increase-323812970dc0e513.yaml | 8 +++ 5 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/image-metadata-size-increase-323812970dc0e513.yaml diff --git a/cinder/api/schemas/volume_image_metadata.py b/cinder/api/schemas/volume_image_metadata.py index 7d810f6693d..7b56caf89f1 100644 --- a/cinder/api/schemas/volume_image_metadata.py +++ b/cinder/api/schemas/volume_image_metadata.py @@ -27,7 +27,7 @@ set_image_metadata = { 'os-set_image_metadata': { 'type': 'object', 'properties': { - 'metadata': parameter_types.extra_specs, + 'metadata': parameter_types.image_metadata, }, 'required': ['metadata'], 'additionalProperties': False, diff --git a/cinder/api/validation/parameter_types.py b/cinder/api/validation/parameter_types.py index 2e098c83ab2..cb8203d548f 100644 --- a/cinder/api/validation/parameter_types.py +++ b/cinder/api/validation/parameter_types.py @@ -157,6 +157,15 @@ extra_specs = { 'additionalProperties': False } +image_metadata = { + 'type': 'object', + 'patternProperties': { + '^[a-zA-Z0-9-_:. /]{1,255}$': { + 'type': 'string', 'format': 'mysql_text' + } + }, + 'additionalProperties': False +} extra_specs_with_no_spaces_key = { 'type': 'object', diff --git a/cinder/api/validation/validators.py b/cinder/api/validation/validators.py index 7369c1ff583..2d714e6ff0d 100644 --- a/cinder/api/validation/validators.py +++ b/cinder/api/validation/validators.py @@ -398,6 +398,14 @@ def _validate_key_size(param_value): return True +@jsonschema.FormatChecker.cls_checks('mysql_text') +def _validate_mysql_text(param_value): + length = len(param_value.encode('utf8')) + if length > 65535: + return False + return True + + class FormatChecker(jsonschema.FormatChecker): """A FormatChecker can output the message from cause exception diff --git a/cinder/tests/unit/api/contrib/test_volume_image_metadata.py b/cinder/tests/unit/api/contrib/test_volume_image_metadata.py index 16b77d75efd..7ca96fc2379 100644 --- a/cinder/tests/unit/api/contrib/test_volume_image_metadata.py +++ b/cinder/tests/unit/api/contrib/test_volume_image_metadata.py @@ -220,6 +220,44 @@ class VolumeImageMetadataTest(test.TestCase): self.assertEqual(fake_image_metadata, jsonutils.loads(res.body)["metadata"]) + # Test for value > 255 + body = {"os-set_image_metadata": { + "metadata": {"key": "v" * 260}} + } + req = webob.Request.blank('/v3/%s/volumes/%s/action' % ( + fake.PROJECT_ID, fake.VOLUME_ID)) + req.method = "POST" + req.body = jsonutils.dump_as_bytes(body) + req.headers["content-type"] = "application/json" + fake_get.return_value = {} + res = req.get_response(fakes.wsgi_app( + fake_auth_context=self.user_ctxt)) + self.assertEqual(HTTPStatus.OK, res.status_int) + + # This is a weird one ... take a supplementary unicode + # char that requires 4 bytes, which will give us a short + # string in terms of character count, but a long string + # in terms of bytes, and make this string be exactly + # 65535 bytes in length. This should be OK. + char4bytes = "\N{CJK UNIFIED IDEOGRAPH-29D98}" + self.assertEqual(1, len(char4bytes)) + self.assertEqual(4, len(char4bytes.encode('utf-8'))) + str65535bytes = char4bytes * 16383 + '123' + self.assertLess(len(str65535bytes), 65535) + self.assertEqual(65535, len(str65535bytes.encode('utf-8'))) + body = {"os-set_image_metadata": { + "metadata": {"key": str65535bytes}} + } + req = webob.Request.blank('/v3/%s/volumes/%s/action' % ( + fake.PROJECT_ID, fake.VOLUME_ID)) + req.method = "POST" + req.body = jsonutils.dump_as_bytes(body) + req.headers["content-type"] = "application/json" + fake_get.return_value = {} + res = req.get_response(fakes.wsgi_app( + fake_auth_context=self.user_ctxt)) + self.assertEqual(HTTPStatus.OK, res.status_int) + @mock.patch('cinder.objects.Volume.get_by_id') def test_create_image_metadata_policy_not_authorized(self, fake_get): rules = { @@ -323,15 +361,38 @@ class VolumeImageMetadataTest(test.TestCase): self.controller.create, req, fake.VOLUME_ID, body=data) - # Test for long value + # Test for very long value data = {"os-set_image_metadata": { - "metadata": {"key": "v" * 260}} + "metadata": {"key": "v" * 65550}} } req.body = jsonutils.dump_as_bytes(data) self.assertRaises(exception.ValidationError, self.controller.create, req, fake.VOLUME_ID, body=data) + # Test for very long utf8 value + data = {"os-set_image_metadata": { + "metadata": {"key": "รก" * 32775}} + } + req.body = jsonutils.dump_as_bytes(data) + self.assertRaises(exception.ValidationError, + self.controller.create, req, fake.VOLUME_ID, + body=data) + + # Test a short unicode string that actually exceeds + # the allowed byte count + char4bytes = "\N{CJK UNIFIED IDEOGRAPH-29D98}" + str65536bytes = char4bytes * 16384 + self.assertEqual(65536, len(str65536bytes.encode('utf-8'))) + self.assertLess(len(str65536bytes), 65535) + body = {"os-set_image_metadata": { + "metadata": {"key": str65536bytes}} + } + req.body = jsonutils.dump_as_bytes(body) + self.assertRaises(exception.ValidationError, + self.controller.create, req, fake.VOLUME_ID, + body=data) + # Test for empty key. data = {"os-set_image_metadata": { "metadata": {"": "value1"}} diff --git a/releasenotes/notes/image-metadata-size-increase-323812970dc0e513.yaml b/releasenotes/notes/image-metadata-size-increase-323812970dc0e513.yaml new file mode 100644 index 00000000000..14be1817d08 --- /dev/null +++ b/releasenotes/notes/image-metadata-size-increase-323812970dc0e513.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + `Bug #1988942 `_: Increased + size of volume image metadata values accepted by the Block Storage API. + Volume image metadata values were limited to 255 characters but Glance + allows up to 65535 bytes. This change does not affect the database + tables which already allow up to 65535 bytes for image metadata values.