openstack/storlets
Code Issues Proposed changes

Fixing headers and user metadata in copy

Browse Source 
This patch fixes a bug that was introduced in the latest
'run copy on object node' change as well as previous bug:

1. One bug was that we called _set_metadata_in_headers
   twice: One from the PUT flow and then again from
   handle_put_copy_response
2. The other bug was calling _set_metadata_in_headers
   in the case of a copy where the execution is not
   done in the proxy, and the object node already
   takes care of the user metadata.
3. We have neglected to copy the get request headers
   coming from the object node before calling the put

Change-Id: I7ea2a9f011a3050f44ee9bf7e117e758ff24c5de
This commit is contained in:
Eran Rom
2017-03-26 13:46:11 +03:00
parent 74aaa7960c
commit f284a69b68
5 changed files with 57 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
StorletSamples/java/ThumbnailStorlet/src/org/openstack/storlet/thumbnail/ThumbnailStorlet.java
View File

@@ -14,11 +14,6 @@
* Limitations under the License.
* ---------------------------------------------------------------------------
*/
/*============================================================================
22-Sep-2014 eranr Initial implementation.
===========================================================================*/
package org.openstack.storlet.thumbnail;
import java.io.IOException;

6
storlets/swift_middleware/handlers/base.py
View File

@@ -384,6 +384,12 @@ class StorletBaseHandler(object):
self.request.params.update(parameters)
def _set_metadata_in_headers(self, headers, user_metadata):
# _set_metadata_in_headers is for user metadata
# set by the storlet invocation. This metadata
# should be prefixed by 'X-Object-Meta' before
# we return things to Swift.
# Do not call this method with metadata headers
# coming from swift request or response.
if user_metadata:
for key, val in user_metadata.items():
headers['X-Object-Meta-%s' % key] = val

34
storlets/swift_middleware/handlers/proxy.py
View File

@@ -19,7 +19,8 @@ try:
# since it is introduced to swift.
from swift.common.middleware.copy import \
_check_copy_from_header as check_copy_from_header, \
_check_destination_header as check_destination_header
_check_destination_header as check_destination_header, \
_copy_headers as copy_headers
except ImportError:
# This is required to keep compatibility with
# swift < 2.8.0 which does not have COPY middleware.
@@ -394,12 +395,11 @@ class StorletProxyHandler(StorletBaseHandler):
'Storlet on copy with %s is not supported' %
header)
def handle_put_copy_response(self, out_md, app_iter):
def handle_put_copy_response(self, app_iter):
self._remove_storlet_headers(self.request.headers)
if 'CONTENT_LENGTH' in self.request.environ:
self.request.environ.pop('CONTENT_LENGTH')
self.request.headers['Transfer-Encoding'] = 'chunked'
self._set_metadata_in_headers(self.request.headers, out_md)
self.request.environ['wsgi.input'] = FileLikeIter(app_iter)
return self.request.get_response(self.app)
@@ -424,25 +424,40 @@ class StorletProxyHandler(StorletBaseHandler):
source_req = self.request.copy_get()
source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
source_req.path_info = source_path
# In the copy case we can run either in the proxy
# or in the object node:
# e.g. if the object is SLO or we have extra resources
# we run on the proxy. Otherwise, we run on the object
# Handling in proxy means that:
# 0. Handle in the proxy
# 1. The object GET request to the object node
# should be called without 'X-Run-Storlet'
# 2. The metadata in the response from the object node
# should not be prefixed with X-Object-Meta
if self.is_proxy_runnable():
# if user set 'X-Storlet-Run-On-Proxy' header, skip invoking at
# object-srever and is_poxy_runnable will be true below
source_req.headers.pop('X-Run-Storlet', None)
src_resp = source_req.get_response(self.app)
copy_headers(src_resp.headers, self.request.headers)
# We check here again, because src_resp may reveal that
# the object is an SLO and so even if the above check was
# False, we now may need to run on proxy
if self.is_proxy_runnable(src_resp):
# We need to run on proxy.
# Do it and fixup the user metadata headers.
sreq = self._build_storlet_request(self.request, src_resp.headers,
src_resp.app_iter)
self.gather_extra_sources()
sresp = self.gateway.invocation_flow(sreq, self.extra_sources)
data_iter = sresp.data_iter
metadata = sresp.user_metadata
self._set_metadata_in_headers(self.request.headers,
sresp.user_metadata)
else:
data_iter = src_resp.app_iter
metadata = src_resp.headers
resp = self.handle_put_copy_response(metadata, data_iter)
resp = self.handle_put_copy_response(data_iter)
acct, path = src_resp.environ['PATH_INFO'].split('/', 3)[2:4]
resp.headers['X-Storlet-Generated-From-Account'] = quote(acct)
@@ -478,8 +493,7 @@ class StorletProxyHandler(StorletBaseHandler):
sresp = self.gateway.invocation_flow(sreq)
self._set_metadata_in_headers(self.request.headers,
sresp.user_metadata)
return self.handle_put_copy_response(sresp.user_metadata,
sresp.data_iter)
return self.handle_put_copy_response(sresp.data_iter)
@public
def COPY(self):

12
tests/functional/java/test_thumbnail_storlet.py
View File

@@ -45,7 +45,8 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
self.assertIn(resp['status'], [200, 202])
def invoke_storlet_on_put(self):
headers = {'X-Run-Storlet': self.storlet_name}
headers = {'X-Run-Storlet': self.storlet_name,
'x-object-meta-name': 'thumbnail'}
headers.update(self.additional_headers)
resp = dict()
source_file = '%s/%s' % (self.path_to_bundle, self.storlet_file)
@@ -61,9 +62,11 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
headers = c.head_object(self.url, self.token,
self.container, 'gen_thumb_on_put.jpg')
self.assertEqual(headers['content-length'], '49032')
self.assertEqual(headers['x-object-meta-name'], 'thumbnail')
def invoke_storlet_on_copy_from(self):
headers = {'X-Run-Storlet': self.storlet_name,
'X-Object-Meta-Name': 'thumbnail',
'X-Copy-From': '%s/%s' %
(self.container, self.storlet_file)}
headers.update(self.additional_headers)
@@ -86,12 +89,16 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
headers = c.head_object(self.url, self.token,
self.container, 'gen_thumb_on_copy.jpg')
self.assertEqual(headers['content-length'], '49032')
self.assertEqual(headers['x-object-meta-name'], 'thumbnail')
self.assertTrue('x-object-meta-x-timestamp' not in headers)
self.assertTrue('x-timestamp' in headers)
def invoke_storlet_on_copy_dest(self):
# No COPY in swiftclient. Using urllib instead...
url = '%s/%s/%s' % (self.url, self.container, self.storlet_file)
headers = {'X-Auth-Token': self.token,
'X-Run-Storlet': self.storlet_name,
'X-Object-Meta-Name': 'thumbnail',
'Destination': '%s/gen_thumb_on_copy_.jpg' % self.container}
headers.update(self.additional_headers)
req = urllib2.Request(url, headers=headers)
@@ -103,6 +110,9 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
headers = c.head_object(self.url, self.token,
self.container, 'gen_thumb_on_copy_.jpg')
self.assertEqual(headers['content-length'], '49032')
self.assertEqual(headers['x-object-meta-name'], 'thumbnail')
self.assertTrue('x-object-meta-x-timestamp' not in headers)
self.assertTrue('x-timestamp' in headers)
def test_get(self):
self.invoke_storlet_on_get()

20
tests/unit/swift_middleware/handlers/test_proxy.py
View File

@@ -298,7 +298,9 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware):
source = '/v1/AUTH_a/c/so'
target = '/v1/AUTH_a/c/to'
copy_from = 'c/so'
self.base_app.register('GET', source, HTTPOk, body='source body')
self.base_app.register('GET', source, HTTPOk,
headers={'x-object-meta-name': 'name'},
body='source body')
self.base_app.register('PUT', target, HTTPCreated)
storlet = '/v1/AUTH_a/storlet/Storlet-1.0.jar'
self.base_app.register('GET', storlet, HTTPOk, body='jar binary')
@@ -317,6 +319,7 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware):
put_calls = self.base_app.get_calls('PUT', target)
self.assertEqual(len(put_calls), 1)
self.assertEqual(put_calls[-1][3], 'source body')
self.assertEqual(put_calls[-1][2]['X-Object-Meta-Name'], 'name')
self.assertNotIn('X-Run-Storlet', put_calls[-1][2])
# no invocation (at gateway stub) at proxy
for debug_line in self.logger.get_log_lines('debug'):
@@ -326,7 +329,9 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware):
source = '/v1/AUTH_a/c/so'
target = '/v1/AUTH_a/c/to'
copy_from = 'c/so'
self.base_app.register('GET', source, HTTPOk, body='source body')
self.base_app.register('GET', source, HTTPOk,
headers={'x-object-meta-name': 'name'},
body='source body')
self.base_app.register('PUT', target, HTTPCreated)
storlet = '/v1/AUTH_a/storlet/Storlet-1.0.jar'
self.base_app.register('GET', storlet, HTTPOk, body='jar binary')
@@ -346,6 +351,7 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware):
put_calls = self.base_app.get_calls('PUT', target)
self.assertEqual(len(put_calls), 1)
self.assertEqual(put_calls[-1][3], 'source body')
self.assertEqual(put_calls[-1][2]['X-Object-Meta-Name'], 'name')
self.assertNotIn('X-Run-Storlet', put_calls[-1][2])
# no invocation at proxy
for debug_line in self.logger.get_log_lines('debug'):
@@ -368,7 +374,9 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware):
source = '/v1/AUTH_a/c/so'
target = '/v1/AUTH_a/c/to'
destination = 'c/to'
self.base_app.register('GET', source, HTTPOk, body='source body')
self.base_app.register('GET', source, HTTPOk,
headers={'x-object-meta-name': 'name'},
body='source body')
self.base_app.register('PUT', target, HTTPCreated)
storlet = '/v1/AUTH_a/storlet/Storlet-1.0.jar'
self.base_app.register('GET', storlet, HTTPOk, body='jar binary')
@@ -387,6 +395,7 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware):
put_calls = self.base_app.get_calls('PUT', target)
self.assertEqual(len(put_calls), 1)
self.assertEqual(put_calls[-1][3], 'source body')
self.assertEqual(put_calls[-1][2]['X-Object-Meta-Name'], 'name')
self.assertNotIn('X-Run-Storlet', put_calls[-1][2])
# no invocation at proxy
for debug_line in self.logger.get_log_lines('debug'):
@@ -396,7 +405,9 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware):
source = '/v1/AUTH_a/c/so'
target = '/v1/AUTH_a/c/to'
destination = 'c/to'
self.base_app.register('GET', source, HTTPOk, body='source body')
self.base_app.register('GET', source, HTTPOk,
headers={'x-object-meta-name': 'name'},
body='source body')
self.base_app.register('PUT', target, HTTPCreated)
storlet = '/v1/AUTH_a/storlet/Storlet-1.0.jar'
self.base_app.register('GET', storlet, HTTPOk, body='jar binary')
@@ -416,6 +427,7 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware):
put_calls = self.base_app.get_calls('PUT', target)
self.assertEqual(len(put_calls), 1)
self.assertEqual(put_calls[-1][3], 'source body')
self.assertEqual(put_calls[-1][2]['X-Object-Meta-Name'], 'name')
self.assertNotIn('X-Run-Storlet', put_calls[-1][2])
# no invocation at proxy
for debug_line in self.logger.get_log_lines('debug'):