openstack
/
storlets
Code Issues Proposed changes

Fix wrong range handling in python storlet execution 

According to HTTP Range spec, we should include the end byte
specified. However, when we execute python storlet in object
server with input range specified, the last one byte is dropped
unexpectedly.

This patch fixes the problem, and make the behavior consistent,
for both of proxy-server execution and object-server execution.

Change-Id: I2fe266a7a21e65472c47701b1f96f4c2c8ee987a
This commit is contained in:
Takashi Kajinami 2017-01-24 23:39:59 +09:00
parent e7a013a2b5
commit 80ca2dacb9
6 changed files with 89 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/java/SCommon/src/main/org/openstack/storlet/common/RangeFileInputStream.java
View File

@ -46,7 +46,7 @@ public class RangeFileInputStream extends FileInputStream {
super(fd);
this.start = start;
this.end = end;
this.available = (int)(end - start) + 1;
this.available = (int)(end - start);
super.skip(start);
}

6
storlets/swift_middleware/handlers/obj.py
View File

@ -54,8 +54,12 @@ class StorletObjectHandler(StorletBaseHandler):
if self.is_storlet_range_request and \
not self.is_storlet_multiple_range_request:
srange = Range(req.headers['X-Storlet-Range'])
# As we should include the end byte in HTTP Range, here we +1
# for the end cursor so that we can treat it as general range
# (include start, and exclude end)
options['range_start'] = srange.ranges[0][0]
options['range_end'] = srange.ranges[0][1]
options['range_end'] = srange.ranges[0][1] + 1
return options

1
tests/functional/java/test_identity_storlet.py
View File

@ -179,6 +179,7 @@ class TestIdentityStorlet(StorletJavaFunctionalTest):
srange = 'bytes=%d-%d' % (start, end)
headers = {'X-Run-Storlet': self.storlet_name,
'X-Storlet-Range': srange}
headers.update(self.additional_headers)
junk, content = c.get_object(self.url, self.token,
'myobjects',
'small',

2
tests/functional/python/test_simple_storlet.py
View File

@ -45,7 +45,7 @@ class TestSimpleStorlet(StorletPythonFunctionalTest):
response_dict=resp, headers=req_headers)
self.assertEqual(200, resp['status'])
self.assertEqual('simple', headers['x-object-meta-test'])
self.assertEqual(self.content[1:4], content)
self.assertEqual(self.content[1:5], content)
def test_put(self):
objname = self.storlet_file + '-put'

53
tests/unit/swift_middleware/handlers/test_obj.py
View File

@ -182,7 +182,7 @@ class TestStorletObjectHandler(unittest.TestCase):
def test_init_handler(self):
req = Request.blank(
'/dev/part/acc/cont/obj', environ={'REQUEST_METHOD': 'GET'},
headers={'X-Backend-Storlet-Policy-Index': '0',
headers={'X-Backend-Storage-Policy-Index': '0',
'X-Run-Storlet': 'Storlet-1.0.jar'})
handler = self.handler_class(
req, self.conf, self.gateway_conf, mock.MagicMock(),
@ -196,7 +196,7 @@ class TestStorletObjectHandler(unittest.TestCase):
req = Request.blank(
'/dev/part/acc2/cont2/obj2', environ={'REQUEST_METHOD': 'GET'},
headers={'X-Backend-Storlet-Policy-Index': '0',
headers={'X-Backend-Storage-Policy-Index': '0',
'X-Run-Storlet': 'Storlet-1.0.jar'})
handler.request = req
self.assertEqual('/dev/part/acc2/cont2/obj2', handler.request.path)
@ -218,6 +218,55 @@ class TestStorletObjectHandler(unittest.TestCase):
with self.assertRaises(AttributeError):
handler.obj = 'obj'
def test_get_storlet_invocation_options(self):
req = Request.blank(
'/dev/part/acc/cont/obj',
environ={'REQUEST_METHOD': 'GET'},
headers={'X-Backend-Storage-Policy-Index': '0',
'X-Run-Storlet': 'Storlet-1.0.jar',
'X-Storlet-Foo': 'baa'})
handler = self.handler_class(
req, self.conf, self.gateway_conf, mock.MagicMock(),
mock.MagicMock())
options = handler._get_storlet_invocation_options(req)
self.assertEqual('baa', options['storlet_foo'])
self.assertFalse(options['generate_log'])
req = Request.blank(
'/dev/part/acc/cont/obj',
environ={'REQUEST_METHOD': 'GET'},
headers={'X-Backend-Storage-Policy-Index': '0',
'X-Run-Storlet': 'Storlet-1.0.jar',
'X-Storlet-Foo': 'baa',
'X-Storlet-Generate-Log': 'True'})
handler = self.handler_class(
req, self.conf, self.gateway_conf, mock.MagicMock(),
mock.MagicMock())
options = handler._get_storlet_invocation_options(req)
self.assertEqual('baa', options['storlet_foo'])
self.assertTrue(options['generate_log'])
req = Request.blank(
'/dev/part/acc/cont/obj',
environ={'REQUEST_METHOD': 'GET'},
headers={'X-Backend-Storage-Policy-Index': '0',
'X-Run-Storlet': 'Storlet-1.0.jar',
'X-Storlet-Foo': 'baa',
'X-Storlet-Range': 'bytes=1-6',
'Range': 'bytes=1-6'})
handler = self.handler_class(
req, self.conf, self.gateway_conf, mock.MagicMock(),
mock.MagicMock())
options = handler._get_storlet_invocation_options(req)
self.assertEqual('baa', options['storlet_foo'])
self.assertFalse(options['generate_log'])
self.assertNotIn('storlet_range', options)
self.assertEqual(1, options['range_start'])
self.assertEqual(7, options['range_end'])
if __name__ == '__main__':
unittest.main()

30
tests/unit/swift_middleware/handlers/test_proxy.py
View File

@ -635,6 +635,36 @@ class TestStorletProxyHandler(unittest.TestCase):
self.assertNotIn('X-Object-Meta-Storlet-Key3', headers)
self.assertEqual(headers['X-Object-Meta-Key4'], 'Value4')
def test_get_storlet_invocation_options(self):
req = Request.blank(
'/v1/acc/cont/obj',
environ={'REQUEST_METHOD': 'GET'},
headers={'X-Run-Storlet': 'Storlet-1.0.jar',
'X-Storlet-Foo': 'baa'})
with storlet_enabled():
handler = self.handler_class(
req, self.conf, self.gateway_conf, mock.MagicMock(),
mock.MagicMock())
options = handler._get_storlet_invocation_options(req)
self.assertEqual('baa', options['storlet_foo'])
self.assertFalse(options['generate_log'])
req = Request.blank(
'/v1/acc/cont/obj',
environ={'REQUEST_METHOD': 'GET'},
headers={'X-Run-Storlet': 'Storlet-1.0.jar',
'X-Storlet-Foo': 'baa',
'X-Storlet-Generate-Log': 'True'})
with storlet_enabled():
handler = self.handler_class(
req, self.conf, self.gateway_conf, mock.MagicMock(),
mock.MagicMock())
options = handler._get_storlet_invocation_options(req)
self.assertEqual('baa', options['storlet_foo'])
self.assertTrue(options['generate_log'])
if __name__ == '__main__':
unittest.main()