Merge "ObjectControllers return application errors as 499 on bad read"

swift/proxy/controllers/obj.py

@@ -69,7 +69,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \
     HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \
     HTTPServerError, HTTPServiceUnavailable, Request, HeaderKeyDict, \
     HTTPClientDisconnect, HTTPUnprocessableEntity, Response, HTTPException, \
-    HTTPRequestedRangeNotSatisfiable, Range
+    HTTPRequestedRangeNotSatisfiable, Range, HTTPInternalServerError
 from swift.common.request_helpers import is_sys_or_user_meta, is_sys_meta, \
     remove_items, copy_header_subset
 
@@ -989,10 +989,15 @@ class ReplicatedObjectController(BaseObjectController):
                 _('Client disconnected without sending last chunk'))
             self.app.logger.increment('client_disconnects')
             raise HTTPClientDisconnect(request=req)
-        except (Exception, Timeout):
+        except Timeout:
             self.app.logger.exception(
                 _('ERROR Exception causing client disconnect'))
             raise HTTPClientDisconnect(request=req)
+        except Exception:
+            self.app.logger.exception(
+                _('ERROR Exception transferring data to object servers %s'),
+                {'path': req.path})
+            raise HTTPInternalServerError(request=req)
         if req.content_length and bytes_transferred < req.content_length:
             req.client_disconnect = True
             self.app.logger.warn(
@@ -2280,10 +2285,15 @@ class ECObjectController(BaseObjectController):
             raise HTTPClientDisconnect(request=req)
         except HTTPException:
             raise
-        except (Exception, Timeout):
+        except Timeout:
             self.app.logger.exception(
                 _('ERROR Exception causing client disconnect'))
             raise HTTPClientDisconnect(request=req)
+        except Exception:
+            self.app.logger.exception(
+                _('ERROR Exception transferring data to object servers %s'),
+                {'path': req.path})
+            raise HTTPInternalServerError(request=req)
 
     def _have_adequate_responses(
             self, statuses, min_responses, conditional_func):

test/unit/proxy/controllers/test_obj.py

@@ -30,7 +30,7 @@ from six import BytesIO
 from six.moves import range
 
 import swift
-from swift.common import utils, swob
+from swift.common import utils, swob, exceptions
 from swift.proxy import server as proxy_server
 from swift.proxy.controllers import obj
 from swift.proxy.controllers.base import get_info as _real_get_info
@@ -612,6 +612,66 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
             node_error_count(self.app, object_ring.devs[1]),
             self.app.error_suppression_limit + 1)
 
+    def test_PUT_error_during_transfer_data(self):
+        class FakeReader(object):
+            def read(self, size):
+                raise exceptions.ChunkReadError(None)
+
+        req = swob.Request.blank('/v1/a/c/o.jpg', method='PUT',
+                                 body='test body')
+
+        req.environ['wsgi.input'] = FakeReader()
+        req.headers['content-length'] = '6'
+        with set_http_connect(201, 201, 201):
+            resp = req.get_response(self.app)
+
+        self.assertEqual(resp.status_int, 499)
+
+    def test_PUT_chunkreadtimeout_during_transfer_data(self):
+        class FakeReader(object):
+            def read(self, size):
+                raise exceptions.ChunkReadTimeout(None)
+
+        req = swob.Request.blank('/v1/a/c/o.jpg', method='PUT',
+                                 body='test body')
+
+        req.environ['wsgi.input'] = FakeReader()
+        req.headers['content-length'] = '6'
+        with set_http_connect(201, 201, 201):
+            resp = req.get_response(self.app)
+
+        self.assertEqual(resp.status_int, 408)
+
+    def test_PUT_timeout_during_transfer_data(self):
+        class FakeReader(object):
+            def read(self, size):
+                raise exceptions.Timeout(None)
+
+        req = swob.Request.blank('/v1/a/c/o.jpg', method='PUT',
+                                 body='test body')
+
+        req.environ['wsgi.input'] = FakeReader()
+        req.headers['content-length'] = '6'
+        with set_http_connect(201, 201, 201):
+            resp = req.get_response(self.app)
+
+        self.assertEqual(resp.status_int, 499)
+
+    def test_PUT_exception_during_transfer_data(self):
+        class FakeReader(object):
+            def read(self, size):
+                raise Exception
+
+        req = swob.Request.blank('/v1/a/c/o.jpg', method='PUT',
+                                 body='test body')
+
+        req.environ['wsgi.input'] = FakeReader()
+        req.headers['content-length'] = '6'
+        with set_http_connect(201, 201, 201):
+            resp = req.get_response(self.app)
+
+        self.assertEqual(resp.status_int, 500)
+
     def test_GET_simple(self):
         req = swift.common.swob.Request.blank('/v1/a/c/o')
         with set_http_connect(200):
@@ -1266,6 +1326,86 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
             resp = req.get_response(self.app)
         self.assertEqual(resp.status_int, 201)
 
+    def test_PUT_ec_error_during_transfer_data(self):
+        class FakeReader(object):
+            def read(self, size):
+                raise exceptions.ChunkReadError(None)
+
+        req = swob.Request.blank('/v1/a/c/o.jpg', method='PUT',
+                                 body='test body')
+
+        req.environ['wsgi.input'] = FakeReader()
+        req.headers['content-length'] = '6'
+        codes = [201] * self.replicas()
+        expect_headers = {
+            'X-Obj-Metadata-Footer': 'yes',
+            'X-Obj-Multiphase-Commit': 'yes'
+        }
+        with set_http_connect(*codes, expect_headers=expect_headers):
+            resp = req.get_response(self.app)
+
+        self.assertEqual(resp.status_int, 499)
+
+    def test_PUT_ec_chunkreadtimeout_during_transfer_data(self):
+        class FakeReader(object):
+            def read(self, size):
+                raise exceptions.ChunkReadTimeout(None)
+
+        req = swob.Request.blank('/v1/a/c/o.jpg', method='PUT',
+                                 body='test body')
+
+        req.environ['wsgi.input'] = FakeReader()
+        req.headers['content-length'] = '6'
+        codes = [201] * self.replicas()
+        expect_headers = {
+            'X-Obj-Metadata-Footer': 'yes',
+            'X-Obj-Multiphase-Commit': 'yes'
+        }
+        with set_http_connect(*codes, expect_headers=expect_headers):
+            resp = req.get_response(self.app)
+
+        self.assertEqual(resp.status_int, 408)
+
+    def test_PUT_ec_timeout_during_transfer_data(self):
+        class FakeReader(object):
+            def read(self, size):
+                raise exceptions.Timeout(None)
+
+        req = swob.Request.blank('/v1/a/c/o.jpg', method='PUT',
+                                 body='test body')
+
+        req.environ['wsgi.input'] = FakeReader()
+        req.headers['content-length'] = '6'
+        codes = [201] * self.replicas()
+        expect_headers = {
+            'X-Obj-Metadata-Footer': 'yes',
+            'X-Obj-Multiphase-Commit': 'yes'
+        }
+        with set_http_connect(*codes, expect_headers=expect_headers):
+            resp = req.get_response(self.app)
+
+        self.assertEqual(resp.status_int, 499)
+
+    def test_PUT_ec_exception_during_transfer_data(self):
+        class FakeReader(object):
+            def read(self, size):
+                raise Exception(None)
+
+        req = swob.Request.blank('/v1/a/c/o.jpg', method='PUT',
+                                 body='test body')
+
+        req.environ['wsgi.input'] = FakeReader()
+        req.headers['content-length'] = '6'
+        codes = [201] * self.replicas()
+        expect_headers = {
+            'X-Obj-Metadata-Footer': 'yes',
+            'X-Obj-Multiphase-Commit': 'yes'
+        }
+        with set_http_connect(*codes, expect_headers=expect_headers):
+            resp = req.get_response(self.app)
+
+        self.assertEqual(resp.status_int, 500)
+
     def test_PUT_with_body(self):
         req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT')
         segment_size = self.policy.ec_segment_size

test/unit/proxy/test_server.py

@@ -3593,17 +3593,17 @@ class TestObjectController(unittest.TestCase):
                 dev['ip'] = '127.0.0.1'
                 dev['port'] = 1
 
-            class SlowBody(object):
+            class DisconnectedBody(object):
 
                 def __init__(self):
                     self.sent = 0
 
                 def read(self, size=-1):
-                    raise Exception('Disconnected')
+                    return ''
 
             req = Request.blank('/v1/a/c/o',
                                 environ={'REQUEST_METHOD': 'PUT',
-                                         'wsgi.input': SlowBody()},
+                                         'wsgi.input': DisconnectedBody()},
                                 headers={'Content-Length': '4',
                                          'Content-Type': 'text/plain'})
             self.app.update_request(req)