Allow chunked image upload in v2 API

* Related to bp api-v2-image-upload

Change-Id: I728cdd28cab1d63d67cd7ad6c9d33535af4d5535

glance/api/v2/image_data.py

@@ -41,15 +41,16 @@ class ImageDataController(base.Controller):
         except exception.NotFound:
             raise webob.exc.HTTPNotFound(_("Image does not exist"))
 
-    def upload(self, req, image_id, data):
+    def upload(self, req, image_id, data, size):
         self._get_image(req.context, image_id)
         try:
             location, size, checksum = self.store_api.add_to_backend(
-                    'file', image_id, data, len(data))
+                    'file', image_id, data, size)
         except exception.Duplicate:
             raise webob.exc.HTTPConflict()
 
-        self.db_api.image_update(req.context, image_id, {'location': location})
+        values = {'location': location, 'size': size}
+        self.db_api.image_update(req.context, image_id, values)
 
     def download(self, req, image_id):
         image = self._get_image(req.context, image_id)
@@ -63,7 +64,13 @@ class ImageDataController(base.Controller):
 
 class RequestDeserializer(wsgi.JSONRequestDeserializer):
     def upload(self, request):
-        return {'data': request.body}
+        try:
+            request.get_content_type('application/octet-stream')
+        except exception.InvalidContentType:
+            raise webob.exc.HTTPUnsupportedMediaType()
+
+        image_size = request.content_length or None
+        return {'size': image_size, 'data': request.body_file}
 
 
 class ResponseSerializer(wsgi.JSONResponseSerializer):

glance/tests/functional/v2/test_images.py

@@ -105,7 +105,7 @@ class TestImages(functional.FunctionalTest):
 
         # Upload some image data
         path = self._url('/v2/images/%s/file' % image_id)
-        headers = self._headers()
+        headers = self._headers({'Content-Type': 'application/octet-stream'})
         response = requests.put(path, headers=headers, data='ZZZZZ')
         self.assertEqual(200, response.status_code)
 
@@ -155,13 +155,13 @@ class TestImages(functional.FunctionalTest):
 
         # Upload some image data
         path = self._url('/v2/images/%s/file' % image_id)
-        headers = self._headers()
+        headers = self._headers({'Content-Type': 'application/octet-stream'})
         response = requests.put(path, headers=headers, data='ZZZZZ')
         self.assertEqual(200, response.status_code)
 
         # Uploading duplicate data should be rejected with a 409
         path = self._url('/v2/images/%s/file' % image_id)
-        headers = self._headers()
+        headers = self._headers({'Content-Type': 'application/octet-stream'})
         response = requests.put(path, headers=headers, data='XXX')
         self.assertEqual(409, response.status_code)
 

glance/tests/unit/utils.py

@@ -16,11 +16,9 @@
 import logging
 import uuid
 
-import webob
-
 import glance.common.context
 from glance.common import exception
-from glance.common import utils
+from glance.common import wsgi
 
 LOG = logging.getLogger(__name__)
 
@@ -34,7 +32,7 @@ USER1 = '54492ba0-f4df-4e4e-be62-27f4d76b29cf'
 USER2 = '0b3b3006-cb76-4517-ae32-51397e22c754'
 
 
-class FakeRequest(webob.Request):
+class FakeRequest(wsgi.Request):
     def __init__(self):
         #TODO(bcwaldon): figure out how to fake this out cleanly
         super(FakeRequest, self).__init__({'REQUEST_METHOD': 'POST'})
@@ -204,6 +202,6 @@ class FakeStoreAPI(object):
     def add_to_backend(self, scheme, image_id, data, size):
         if image_id in self.data:
             raise exception.Duplicate()
-        self.data[image_id] = (data, size)
+        self.data[image_id] = (data, size or len(data))
         checksum = 'Z'
         return (image_id, size, checksum)

glance/tests/unit/v2/test_image_data_resource.py

@@ -13,6 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import StringIO
 import unittest
 
 import webob
@@ -53,7 +54,7 @@ class TestImagesController(unittest.TestCase):
 
     def test_upload_download(self):
         request = test_utils.FakeRequest()
-        self.controller.upload(request, test_utils.UUID2, 'YYYY')
+        self.controller.upload(request, test_utils.UUID2, 'YYYY', 4)
         output = self.controller.download(request, test_utils.UUID2)
         expected = {'data': 'YYYY', 'size': 4}
         self.assertEqual(expected, output)
@@ -61,12 +62,19 @@ class TestImagesController(unittest.TestCase):
     def test_upload_non_existant_image(self):
         request = test_utils.FakeRequest()
         self.assertRaises(webob.exc.HTTPNotFound, self.controller.upload,
-                          request, utils.generate_uuid(), 'YYYY')
+                          request, utils.generate_uuid(), 'YYYY', 4)
 
     def test_upload_data_exists(self):
         request = test_utils.FakeRequest()
         self.assertRaises(webob.exc.HTTPConflict, self.controller.upload,
-                          request, test_utils.UUID1, 'YYYY')
+                          request, test_utils.UUID1, 'YYYY', 4)
+
+    def test_upload_download_no_size(self):
+        request = test_utils.FakeRequest()
+        self.controller.upload(request, test_utils.UUID2, 'YYYY', None)
+        output = self.controller.download(request, test_utils.UUID2)
+        expected = {'data': 'YYYY', 'size': 4}
+        self.assertEqual(expected, output)
 
 
 class TestImageDataDeserializer(unittest.TestCase):
@@ -75,11 +83,61 @@ class TestImageDataDeserializer(unittest.TestCase):
 
     def test_upload(self):
         request = test_utils.FakeRequest()
+        request.headers['Content-Type'] = 'application/octet-stream'
         request.body = 'YYY'
+        request.headers['Content-Length'] = 3
         output = self.deserializer.upload(request)
-        expected = {'data': 'YYY'}
+        data = output.pop('data')
+        self.assertEqual(data.getvalue(), 'YYY')
+        expected = {'size': 3}
         self.assertEqual(expected, output)
 
+    def test_upload_chunked(self):
+        request = test_utils.FakeRequest()
+        request.headers['Content-Type'] = 'application/octet-stream'
+        # If we use body_file, webob assumes we want to do a chunked upload,
+        # ignoring the Content-Length header
+        request.body_file = StringIO.StringIO('YYY')
+        output = self.deserializer.upload(request)
+        data = output.pop('data')
+        self.assertEqual(data.getvalue(), 'YYY')
+        expected = {'size': None}
+        self.assertEqual(expected, output)
+
+    def test_upload_chunked_with_content_length(self):
+        request = test_utils.FakeRequest()
+        request.headers['Content-Type'] = 'application/octet-stream'
+        request.body_file = StringIO.StringIO('YYY')
+        # The deserializer shouldn't care if the Content-Length is
+        # set when the user is attempting to send chunked data.
+        request.headers['Content-Length'] = 3
+        output = self.deserializer.upload(request)
+        data = output.pop('data')
+        self.assertEqual(data.getvalue(), 'YYY')
+        expected = {'size': 3}
+        self.assertEqual(expected, output)
+
+    def test_upload_with_incorrect_content_length(self):
+        request = test_utils.FakeRequest()
+        request.headers['Content-Type'] = 'application/octet-stream'
+        # The deserializer shouldn't care if the Content-Length and
+        # actual request body length differ. That job is left up
+        # to the controller
+        request.body = 'YYY'
+        request.headers['Content-Length'] = 4
+        output = self.deserializer.upload(request)
+        data = output.pop('data')
+        self.assertEqual(data.getvalue(), 'YYY')
+        expected = {'size': 4}
+        self.assertEqual(expected, output)
+
+    def test_upload_wrong_content_type(self):
+        request = test_utils.FakeRequest()
+        request.headers['Content-Type'] = 'application/json'
+        request.body = 'YYYYY'
+        self.assertRaises(webob.exc.HTTPUnsupportedMediaType,
+            self.deserializer.upload, request)
+
 
 class TestImageDataSerializer(unittest.TestCase):
     def setUp(self):