Read image size from request header

Image upload and image stage APIs now reads the request header
`x-openstack-image-size` and pass the image size to actual storage
bakend while uploading the image data. This change now also
set image size in the database even before image upload or
staging operation starts. As per backend implementation
if actual image size mismatches with the size proivded in
request header then the process will be aborted and HTTP 400
response will be returned to the user.

Depends-On: https://review.opendev.org/c/openstack/python-glanceclient/+/949787

Implements: blueprint set-size-on-upload

Change-Id: Ib2c282bc19d48361b16fb1b2549cfcab80dea29c
Signed-off-by: Abhishek Kekane <akekane@redhat.com>

glance/api/v2/image_data.py

@@ -278,6 +278,13 @@ class ImageDataController(object):
             raise webob.exc.HTTPServiceUnavailable(explanation=msg,
                                                    request=req)
 
+        except glance_store.Invalid as e:
+            LOG.error(e.message)
+            if image.status not in ('queued', 'deleted'):
+                self._restore(image_repo, image)
+            raise webob.exc.HTTPBadRequest(
+                explanation=str(e))
+
         except cursive_exception.SignatureVerificationError as e:
             msg = (_LE("Signature verification failed for image %(id)s: %(e)s")
                    % {'id': image_id, 'e': e})
@@ -302,6 +309,8 @@ class ImageDataController(object):
 
     @utils.mutating
     def stage(self, req, image_id, data, size):
+        if size is None:
+            size = 0
         try:
             ks_quota.enforce_image_staging_total(req.context,
                                                  req.context.owner)
@@ -368,10 +377,11 @@ class ImageDataController(object):
             ks_quota.enforce_image_count_uploading(req.context,
                                                    req.context.owner)
             try:
-                uri, size, id, store_info = staging_store.add(
+                uri, image_size, id, store_info = staging_store.add(
                     image_id, utils.LimitingReader(
-                        utils.CooperativeReader(data), CONF.image_size_cap), 0)
+                        utils.CooperativeReader(data), CONF.image_size_cap),
-                image.size = size
+                    size)
+                image.size = image_size
             except glance_store.Duplicate:
                 msg = _("The image %s has data on staging") % image_id
                 raise webob.exc.HTTPConflict(explanation=msg)
@@ -394,6 +404,13 @@ class ImageDataController(object):
             raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
                                                       request=req)
 
+        except glance_store.Invalid as e:
+            LOG.error(e.message)
+            if image.status not in ('queued', 'deleted'):
+                self._restore(image_repo, image)
+            raise webob.exc.HTTPBadRequest(
+                explanation=str(e))
+
         except exception.StorageQuotaFull as e:
             msg = _("Image exceeds the storage quota: %s") % e
             LOG.debug(msg)
@@ -456,6 +473,20 @@ class ImageDataController(object):
 
 class RequestDeserializer(wsgi.JSONRequestDeserializer):
 
+    def _get_image_size(self, request):
+        try:
+            size = request.headers.get('x-openstack-image-size')
+            if size is not None:
+                image_size = int(size)
+            else:
+                # If header is missing, fall back to content_length or None
+                image_size = request.content_length or None
+        except (ValueError, TypeError):
+            # Raised if conversion to int fails or value is None
+            raise webob.exc.HTTPBadRequest(explanation=_(
+                "Invalid or missing image size in request headers."))
+        return image_size
+
     def upload(self, request):
         try:
             request.get_content_type(('application/octet-stream',))
@@ -465,8 +496,8 @@ class RequestDeserializer(wsgi.JSONRequestDeserializer):
         if self.is_valid_encoding(request) and self.is_valid_method(request):
             request.is_body_readable = True
 
-        image_size = request.content_length or None
+        return {'size': self._get_image_size(request),
-        return {'size': image_size, 'data': request.body_file}
+                'data': request.body_file}
 
     def stage(self, request):
         if "glance-direct" not in CONF.enabled_import_methods:
@@ -480,8 +511,8 @@ class RequestDeserializer(wsgi.JSONRequestDeserializer):
         if self.is_valid_encoding(request) and self.is_valid_method(request):
             request.is_body_readable = True
 
-        image_size = request.content_length or None
+        return {'size': self._get_image_size(request),
-        return {'size': image_size, 'data': request.body_file}
+                'data': request.body_file}
 
 
 class ResponseSerializer(wsgi.JSONResponseSerializer):

glance/tests/unit/v2/test_image_data_resource.py

@@ -80,6 +80,10 @@ class FakeImage(object):
 
     def set_data(self, data, size=None, backend=None, set_active=True):
         self.data = ''.join(data)
+        if not size:
+            size = len(self.data)
+        if size != len(self.data):
+            raise webob.exc.HTTPBadRequest()
         self.size = size
         self.status = 'modified-by-fake'
 
@@ -98,6 +102,9 @@ class FakeImageRepo(object):
     def save(self, image, from_state=None):
         self.saved_image = image
 
+    def list(self):
+        return []
+
 
 class FakeGateway(object):
 
@@ -239,7 +246,21 @@ class TestImagesController(base.StoreClearingUnitTest):
         self.image_repo.result = image
         self.controller.upload(request, unit_test_utils.UUID2, 'YYYY', None)
         self.assertEqual('YYYY', image.data)
-        self.assertIsNone(image.size)
+        self.assertEqual(4, image.size)
+
+    def test_upload_size_more_than_data(self):
+        request = unit_test_utils.get_fake_request(roles=['admin', 'member'])
+        image = FakeImage('abcd')
+        self.image_repo.result = image
+        self.assertRaises(webob.exc.HTTPBadRequest, self.controller.upload,
+                          request, unit_test_utils.UUID2, 'YYYY', 5)
+
+    def test_upload_size_less_than_data(self):
+        request = unit_test_utils.get_fake_request(roles=['admin', 'member'])
+        image = FakeImage('abcd')
+        self.image_repo.result = image
+        self.assertRaises(webob.exc.HTTPBadRequest, self.controller.upload,
+                          request, unit_test_utils.UUID2, 'YYYY', 2)
 
     @mock.patch.object(glance.api.policy.Enforcer, 'enforce')
     def test_upload_image_forbidden(self, mock_enforce):
@@ -520,6 +541,35 @@ class TestImagesController(base.StoreClearingUnitTest):
         self.assertEqual('uploading', image.status)
         self.assertEqual(4, image.size)
 
+    def test_stage_no_size(self):
+        image_id = str(uuid.uuid4())
+        request = unit_test_utils.get_fake_request(roles=['admin', 'member'])
+        image = FakeImage(image_id=image_id)
+        self.image_repo.result = image
+        with mock.patch.object(filesystem.Store, 'add') as mock_add:
+            mock_add.return_value = ('foo://bar', 4, 'ident', {})
+            self.controller.stage(request, image_id, 'YYYY', None)
+        self.assertEqual('uploading', image.status)
+        self.assertEqual(4, image.size)
+
+    def test_stage_size_more_than_data(self):
+        request = unit_test_utils.get_fake_request(roles=['admin', 'member'])
+        image = FakeImage('abcd')
+        self.image_repo.result = image
+        with mock.patch.object(filesystem.Store, 'add') as mock_add:
+            mock_add.side_effect = glance_store.Invalid()
+            self.assertRaises(webob.exc.HTTPBadRequest, self.controller.stage,
+                              request, unit_test_utils.UUID2, 'YYYY', 5)
+
+    def test_stage_size_less_than_data(self):
+        request = unit_test_utils.get_fake_request(roles=['admin', 'member'])
+        image = FakeImage('abcd')
+        self.image_repo.result = image
+        with mock.patch.object(filesystem.Store, 'add') as mock_add:
+            mock_add.side_effect = glance_store.Invalid()
+            self.assertRaises(webob.exc.HTTPBadRequest, self.controller.stage,
+                              request, unit_test_utils.UUID2, 'YYYY', 2)
+
     def test_image_already_on_staging(self):
         image_id = str(uuid.uuid4())
         request = unit_test_utils.get_fake_request(roles=['admin', 'member'])
@@ -760,6 +810,57 @@ class TestImageDataDeserializer(test_utils.BaseTestCase):
                           self.deserializer.stage,
                           req)
 
+    def test_with_size_header(self):
+        for method in ('upload', 'stage'):
+            with self.subTest(method=method):
+                request = unit_test_utils.get_fake_request()
+                request.headers['Content-Type'] = 'application/octet-stream'
+                request.headers['x-openstack-image-size'] = 4
+                request.body = b'YYYY'
+                func = getattr(self.deserializer, method)
+                output = func(request)
+                data = output.pop('data')
+                self.assertEqual(b'YYYY', data.read())
+                expected = {'size': 4}
+                self.assertEqual(expected, output)
+
+    def test_without_size_header_and_content_length(self):
+        for method in ('upload', 'stage'):
+            with self.subTest(method=method):
+                request = unit_test_utils.get_fake_request()
+                request.headers['Content-Type'] = 'application/octet-stream'
+                request.body_file = io.StringIO('YYYY')
+                func = getattr(self.deserializer, method)
+                output = func(request)
+                data = output.pop('data')
+                self.assertEqual('YYYY', data.read())
+                expected = {'size': None}
+                self.assertEqual(expected, output)
+
+    def test_size_header_raising_type_error(self):
+        for method in ('upload', 'stage'):
+            with self.subTest(method=method):
+                request = unit_test_utils.get_fake_request()
+                request.headers['Content-Type'] = 'application/octet-stream'
+                request.headers['x-openstack-image-size'] = [4]
+                request.body = b'YYYY'
+                func = getattr(self.deserializer, method)
+                self.assertRaisesRegex(
+                    webob.exc.HTTPBadRequest, "Invalid or missing image size",
+                    func, request)
+
+    def test_with_invalid_size_header(self):
+        for method in ('upload', 'stage'):
+            with self.subTest(method=method):
+                request = unit_test_utils.get_fake_request()
+                request.headers['Content-Type'] = 'application/octet-stream'
+                request.headers['x-openstack-image-size'] = 'foobar'
+                request.body = b'YYYY'
+                func = getattr(self.deserializer, method)
+                self.assertRaisesRegex(
+                    webob.exc.HTTPBadRequest, "Invalid or missing image size",
+                    func, request)
+
 
 class TestImageDataSerializer(test_utils.BaseTestCase):