From 7f761e16ce69ec549be1a66bd2797e53d0a1d988 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Tue, 8 May 2012 15:39:00 -0700 Subject: [PATCH] Allow chunked image upload in v2 API * Related to bp api-v2-image-upload Change-Id: I728cdd28cab1d63d67cd7ad6c9d33535af4d5535 --- glance/api/v2/image_data.py | 15 +++-- glance/tests/functional/v2/test_images.py | 6 +- glance/tests/unit/utils.py | 8 +-- .../tests/unit/v2/test_image_data_resource.py | 66 +++++++++++++++++-- 4 files changed, 79 insertions(+), 16 deletions(-) diff --git a/glance/api/v2/image_data.py b/glance/api/v2/image_data.py index 3a31d04ef3..c1e851f4a8 100644 --- a/glance/api/v2/image_data.py +++ b/glance/api/v2/image_data.py @@ -41,15 +41,16 @@ class ImageDataController(base.Controller): except exception.NotFound: raise webob.exc.HTTPNotFound(_("Image does not exist")) - def upload(self, req, image_id, data): + def upload(self, req, image_id, data, size): self._get_image(req.context, image_id) try: location, size, checksum = self.store_api.add_to_backend( - 'file', image_id, data, len(data)) + 'file', image_id, data, size) except exception.Duplicate: raise webob.exc.HTTPConflict() - self.db_api.image_update(req.context, image_id, {'location': location}) + values = {'location': location, 'size': size} + self.db_api.image_update(req.context, image_id, values) def download(self, req, image_id): image = self._get_image(req.context, image_id) @@ -63,7 +64,13 @@ class ImageDataController(base.Controller): class RequestDeserializer(wsgi.JSONRequestDeserializer): def upload(self, request): - return {'data': request.body} + try: + request.get_content_type('application/octet-stream') + except exception.InvalidContentType: + raise webob.exc.HTTPUnsupportedMediaType() + + image_size = request.content_length or None + return {'size': image_size, 'data': request.body_file} class ResponseSerializer(wsgi.JSONResponseSerializer): diff --git a/glance/tests/functional/v2/test_images.py b/glance/tests/functional/v2/test_images.py index 20bbfb1c1f..a200e88db4 100644 --- a/glance/tests/functional/v2/test_images.py +++ b/glance/tests/functional/v2/test_images.py @@ -105,7 +105,7 @@ class TestImages(functional.FunctionalTest): # Upload some image data path = self._url('/v2/images/%s/file' % image_id) - headers = self._headers() + headers = self._headers({'Content-Type': 'application/octet-stream'}) response = requests.put(path, headers=headers, data='ZZZZZ') self.assertEqual(200, response.status_code) @@ -155,13 +155,13 @@ class TestImages(functional.FunctionalTest): # Upload some image data path = self._url('/v2/images/%s/file' % image_id) - headers = self._headers() + headers = self._headers({'Content-Type': 'application/octet-stream'}) response = requests.put(path, headers=headers, data='ZZZZZ') self.assertEqual(200, response.status_code) # Uploading duplicate data should be rejected with a 409 path = self._url('/v2/images/%s/file' % image_id) - headers = self._headers() + headers = self._headers({'Content-Type': 'application/octet-stream'}) response = requests.put(path, headers=headers, data='XXX') self.assertEqual(409, response.status_code) diff --git a/glance/tests/unit/utils.py b/glance/tests/unit/utils.py index e646390aa4..2a6c224cbe 100644 --- a/glance/tests/unit/utils.py +++ b/glance/tests/unit/utils.py @@ -16,11 +16,9 @@ import logging import uuid -import webob - import glance.common.context from glance.common import exception -from glance.common import utils +from glance.common import wsgi LOG = logging.getLogger(__name__) @@ -34,7 +32,7 @@ USER1 = '54492ba0-f4df-4e4e-be62-27f4d76b29cf' USER2 = '0b3b3006-cb76-4517-ae32-51397e22c754' -class FakeRequest(webob.Request): +class FakeRequest(wsgi.Request): def __init__(self): #TODO(bcwaldon): figure out how to fake this out cleanly super(FakeRequest, self).__init__({'REQUEST_METHOD': 'POST'}) @@ -204,6 +202,6 @@ class FakeStoreAPI(object): def add_to_backend(self, scheme, image_id, data, size): if image_id in self.data: raise exception.Duplicate() - self.data[image_id] = (data, size) + self.data[image_id] = (data, size or len(data)) checksum = 'Z' return (image_id, size, checksum) diff --git a/glance/tests/unit/v2/test_image_data_resource.py b/glance/tests/unit/v2/test_image_data_resource.py index 0838006ad4..4aeb02bd48 100644 --- a/glance/tests/unit/v2/test_image_data_resource.py +++ b/glance/tests/unit/v2/test_image_data_resource.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +import StringIO import unittest import webob @@ -53,7 +54,7 @@ class TestImagesController(unittest.TestCase): def test_upload_download(self): request = test_utils.FakeRequest() - self.controller.upload(request, test_utils.UUID2, 'YYYY') + self.controller.upload(request, test_utils.UUID2, 'YYYY', 4) output = self.controller.download(request, test_utils.UUID2) expected = {'data': 'YYYY', 'size': 4} self.assertEqual(expected, output) @@ -61,12 +62,19 @@ class TestImagesController(unittest.TestCase): def test_upload_non_existant_image(self): request = test_utils.FakeRequest() self.assertRaises(webob.exc.HTTPNotFound, self.controller.upload, - request, utils.generate_uuid(), 'YYYY') + request, utils.generate_uuid(), 'YYYY', 4) def test_upload_data_exists(self): request = test_utils.FakeRequest() self.assertRaises(webob.exc.HTTPConflict, self.controller.upload, - request, test_utils.UUID1, 'YYYY') + request, test_utils.UUID1, 'YYYY', 4) + + def test_upload_download_no_size(self): + request = test_utils.FakeRequest() + self.controller.upload(request, test_utils.UUID2, 'YYYY', None) + output = self.controller.download(request, test_utils.UUID2) + expected = {'data': 'YYYY', 'size': 4} + self.assertEqual(expected, output) class TestImageDataDeserializer(unittest.TestCase): @@ -75,11 +83,61 @@ class TestImageDataDeserializer(unittest.TestCase): def test_upload(self): request = test_utils.FakeRequest() + request.headers['Content-Type'] = 'application/octet-stream' request.body = 'YYY' + request.headers['Content-Length'] = 3 output = self.deserializer.upload(request) - expected = {'data': 'YYY'} + data = output.pop('data') + self.assertEqual(data.getvalue(), 'YYY') + expected = {'size': 3} self.assertEqual(expected, output) + def test_upload_chunked(self): + request = test_utils.FakeRequest() + request.headers['Content-Type'] = 'application/octet-stream' + # If we use body_file, webob assumes we want to do a chunked upload, + # ignoring the Content-Length header + request.body_file = StringIO.StringIO('YYY') + output = self.deserializer.upload(request) + data = output.pop('data') + self.assertEqual(data.getvalue(), 'YYY') + expected = {'size': None} + self.assertEqual(expected, output) + + def test_upload_chunked_with_content_length(self): + request = test_utils.FakeRequest() + request.headers['Content-Type'] = 'application/octet-stream' + request.body_file = StringIO.StringIO('YYY') + # The deserializer shouldn't care if the Content-Length is + # set when the user is attempting to send chunked data. + request.headers['Content-Length'] = 3 + output = self.deserializer.upload(request) + data = output.pop('data') + self.assertEqual(data.getvalue(), 'YYY') + expected = {'size': 3} + self.assertEqual(expected, output) + + def test_upload_with_incorrect_content_length(self): + request = test_utils.FakeRequest() + request.headers['Content-Type'] = 'application/octet-stream' + # The deserializer shouldn't care if the Content-Length and + # actual request body length differ. That job is left up + # to the controller + request.body = 'YYY' + request.headers['Content-Length'] = 4 + output = self.deserializer.upload(request) + data = output.pop('data') + self.assertEqual(data.getvalue(), 'YYY') + expected = {'size': 4} + self.assertEqual(expected, output) + + def test_upload_wrong_content_type(self): + request = test_utils.FakeRequest() + request.headers['Content-Type'] = 'application/json' + request.body = 'YYYYY' + self.assertRaises(webob.exc.HTTPUnsupportedMediaType, + self.deserializer.upload, request) + class TestImageDataSerializer(unittest.TestCase): def setUp(self):