diff --git a/StorletSamples/java/ThumbnailStorlet/src/org/openstack/storlet/thumbnail/ThumbnailStorlet.java b/StorletSamples/java/ThumbnailStorlet/src/org/openstack/storlet/thumbnail/ThumbnailStorlet.java index a37f8106..2e10039e 100644 --- a/StorletSamples/java/ThumbnailStorlet/src/org/openstack/storlet/thumbnail/ThumbnailStorlet.java +++ b/StorletSamples/java/ThumbnailStorlet/src/org/openstack/storlet/thumbnail/ThumbnailStorlet.java @@ -14,11 +14,6 @@ * Limitations under the License. * --------------------------------------------------------------------------- */ - -/*============================================================================ - 22-Sep-2014 eranr Initial implementation. - ===========================================================================*/ - package org.openstack.storlet.thumbnail; import java.io.IOException; diff --git a/storlets/swift_middleware/handlers/base.py b/storlets/swift_middleware/handlers/base.py index c71cf827..8bdecd94 100644 --- a/storlets/swift_middleware/handlers/base.py +++ b/storlets/swift_middleware/handlers/base.py @@ -384,6 +384,12 @@ class StorletBaseHandler(object): self.request.params.update(parameters) def _set_metadata_in_headers(self, headers, user_metadata): + # _set_metadata_in_headers is for user metadata + # set by the storlet invocation. This metadata + # should be prefixed by 'X-Object-Meta' before + # we return things to Swift. + # Do not call this method with metadata headers + # coming from swift request or response. if user_metadata: for key, val in user_metadata.items(): headers['X-Object-Meta-%s' % key] = val diff --git a/storlets/swift_middleware/handlers/proxy.py b/storlets/swift_middleware/handlers/proxy.py index 1854f772..f0d760f3 100644 --- a/storlets/swift_middleware/handlers/proxy.py +++ b/storlets/swift_middleware/handlers/proxy.py @@ -19,7 +19,8 @@ try: # since it is introduced to swift. from swift.common.middleware.copy import \ _check_copy_from_header as check_copy_from_header, \ - _check_destination_header as check_destination_header + _check_destination_header as check_destination_header, \ + _copy_headers as copy_headers except ImportError: # This is required to keep compatibility with # swift < 2.8.0 which does not have COPY middleware. @@ -394,12 +395,11 @@ class StorletProxyHandler(StorletBaseHandler): 'Storlet on copy with %s is not supported' % header) - def handle_put_copy_response(self, out_md, app_iter): + def handle_put_copy_response(self, app_iter): self._remove_storlet_headers(self.request.headers) if 'CONTENT_LENGTH' in self.request.environ: self.request.environ.pop('CONTENT_LENGTH') self.request.headers['Transfer-Encoding'] = 'chunked' - self._set_metadata_in_headers(self.request.headers, out_md) self.request.environ['wsgi.input'] = FileLikeIter(app_iter) return self.request.get_response(self.app) @@ -424,25 +424,40 @@ class StorletProxyHandler(StorletBaseHandler): source_req = self.request.copy_get() source_req.headers.pop('X-Backend-Storage-Policy-Index', None) source_req.path_info = source_path + + # In the copy case we can run either in the proxy + # or in the object node: + # e.g. if the object is SLO or we have extra resources + # we run on the proxy. Otherwise, we run on the object + # Handling in proxy means that: + # 0. Handle in the proxy + # 1. The object GET request to the object node + # should be called without 'X-Run-Storlet' + # 2. The metadata in the response from the object node + # should not be prefixed with X-Object-Meta if self.is_proxy_runnable(): - # if user set 'X-Storlet-Run-On-Proxy' header, skip invoking at - # object-srever and is_poxy_runnable will be true below source_req.headers.pop('X-Run-Storlet', None) src_resp = source_req.get_response(self.app) + copy_headers(src_resp.headers, self.request.headers) + # We check here again, because src_resp may reveal that + # the object is an SLO and so even if the above check was + # False, we now may need to run on proxy if self.is_proxy_runnable(src_resp): + # We need to run on proxy. + # Do it and fixup the user metadata headers. sreq = self._build_storlet_request(self.request, src_resp.headers, src_resp.app_iter) self.gather_extra_sources() sresp = self.gateway.invocation_flow(sreq, self.extra_sources) data_iter = sresp.data_iter - metadata = sresp.user_metadata + self._set_metadata_in_headers(self.request.headers, + sresp.user_metadata) else: data_iter = src_resp.app_iter - metadata = src_resp.headers - resp = self.handle_put_copy_response(metadata, data_iter) + resp = self.handle_put_copy_response(data_iter) acct, path = src_resp.environ['PATH_INFO'].split('/', 3)[2:4] resp.headers['X-Storlet-Generated-From-Account'] = quote(acct) @@ -478,8 +493,7 @@ class StorletProxyHandler(StorletBaseHandler): sresp = self.gateway.invocation_flow(sreq) self._set_metadata_in_headers(self.request.headers, sresp.user_metadata) - return self.handle_put_copy_response(sresp.user_metadata, - sresp.data_iter) + return self.handle_put_copy_response(sresp.data_iter) @public def COPY(self): diff --git a/tests/functional/java/test_thumbnail_storlet.py b/tests/functional/java/test_thumbnail_storlet.py index 0f5b7e5b..133f797d 100644 --- a/tests/functional/java/test_thumbnail_storlet.py +++ b/tests/functional/java/test_thumbnail_storlet.py @@ -45,7 +45,8 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest): self.assertIn(resp['status'], [200, 202]) def invoke_storlet_on_put(self): - headers = {'X-Run-Storlet': self.storlet_name} + headers = {'X-Run-Storlet': self.storlet_name, + 'x-object-meta-name': 'thumbnail'} headers.update(self.additional_headers) resp = dict() source_file = '%s/%s' % (self.path_to_bundle, self.storlet_file) @@ -61,9 +62,11 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest): headers = c.head_object(self.url, self.token, self.container, 'gen_thumb_on_put.jpg') self.assertEqual(headers['content-length'], '49032') + self.assertEqual(headers['x-object-meta-name'], 'thumbnail') def invoke_storlet_on_copy_from(self): headers = {'X-Run-Storlet': self.storlet_name, + 'X-Object-Meta-Name': 'thumbnail', 'X-Copy-From': '%s/%s' % (self.container, self.storlet_file)} headers.update(self.additional_headers) @@ -86,12 +89,16 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest): headers = c.head_object(self.url, self.token, self.container, 'gen_thumb_on_copy.jpg') self.assertEqual(headers['content-length'], '49032') + self.assertEqual(headers['x-object-meta-name'], 'thumbnail') + self.assertTrue('x-object-meta-x-timestamp' not in headers) + self.assertTrue('x-timestamp' in headers) def invoke_storlet_on_copy_dest(self): # No COPY in swiftclient. Using urllib instead... url = '%s/%s/%s' % (self.url, self.container, self.storlet_file) headers = {'X-Auth-Token': self.token, 'X-Run-Storlet': self.storlet_name, + 'X-Object-Meta-Name': 'thumbnail', 'Destination': '%s/gen_thumb_on_copy_.jpg' % self.container} headers.update(self.additional_headers) req = urllib2.Request(url, headers=headers) @@ -103,6 +110,9 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest): headers = c.head_object(self.url, self.token, self.container, 'gen_thumb_on_copy_.jpg') self.assertEqual(headers['content-length'], '49032') + self.assertEqual(headers['x-object-meta-name'], 'thumbnail') + self.assertTrue('x-object-meta-x-timestamp' not in headers) + self.assertTrue('x-timestamp' in headers) def test_get(self): self.invoke_storlet_on_get() diff --git a/tests/unit/swift_middleware/handlers/test_proxy.py b/tests/unit/swift_middleware/handlers/test_proxy.py index e6896a2a..9e91bca0 100644 --- a/tests/unit/swift_middleware/handlers/test_proxy.py +++ b/tests/unit/swift_middleware/handlers/test_proxy.py @@ -298,7 +298,9 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware): source = '/v1/AUTH_a/c/so' target = '/v1/AUTH_a/c/to' copy_from = 'c/so' - self.base_app.register('GET', source, HTTPOk, body='source body') + self.base_app.register('GET', source, HTTPOk, + headers={'x-object-meta-name': 'name'}, + body='source body') self.base_app.register('PUT', target, HTTPCreated) storlet = '/v1/AUTH_a/storlet/Storlet-1.0.jar' self.base_app.register('GET', storlet, HTTPOk, body='jar binary') @@ -317,6 +319,7 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware): put_calls = self.base_app.get_calls('PUT', target) self.assertEqual(len(put_calls), 1) self.assertEqual(put_calls[-1][3], 'source body') + self.assertEqual(put_calls[-1][2]['X-Object-Meta-Name'], 'name') self.assertNotIn('X-Run-Storlet', put_calls[-1][2]) # no invocation (at gateway stub) at proxy for debug_line in self.logger.get_log_lines('debug'): @@ -326,7 +329,9 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware): source = '/v1/AUTH_a/c/so' target = '/v1/AUTH_a/c/to' copy_from = 'c/so' - self.base_app.register('GET', source, HTTPOk, body='source body') + self.base_app.register('GET', source, HTTPOk, + headers={'x-object-meta-name': 'name'}, + body='source body') self.base_app.register('PUT', target, HTTPCreated) storlet = '/v1/AUTH_a/storlet/Storlet-1.0.jar' self.base_app.register('GET', storlet, HTTPOk, body='jar binary') @@ -346,6 +351,7 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware): put_calls = self.base_app.get_calls('PUT', target) self.assertEqual(len(put_calls), 1) self.assertEqual(put_calls[-1][3], 'source body') + self.assertEqual(put_calls[-1][2]['X-Object-Meta-Name'], 'name') self.assertNotIn('X-Run-Storlet', put_calls[-1][2]) # no invocation at proxy for debug_line in self.logger.get_log_lines('debug'): @@ -368,7 +374,9 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware): source = '/v1/AUTH_a/c/so' target = '/v1/AUTH_a/c/to' destination = 'c/to' - self.base_app.register('GET', source, HTTPOk, body='source body') + self.base_app.register('GET', source, HTTPOk, + headers={'x-object-meta-name': 'name'}, + body='source body') self.base_app.register('PUT', target, HTTPCreated) storlet = '/v1/AUTH_a/storlet/Storlet-1.0.jar' self.base_app.register('GET', storlet, HTTPOk, body='jar binary') @@ -387,6 +395,7 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware): put_calls = self.base_app.get_calls('PUT', target) self.assertEqual(len(put_calls), 1) self.assertEqual(put_calls[-1][3], 'source body') + self.assertEqual(put_calls[-1][2]['X-Object-Meta-Name'], 'name') self.assertNotIn('X-Run-Storlet', put_calls[-1][2]) # no invocation at proxy for debug_line in self.logger.get_log_lines('debug'): @@ -396,7 +405,9 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware): source = '/v1/AUTH_a/c/so' target = '/v1/AUTH_a/c/to' destination = 'c/to' - self.base_app.register('GET', source, HTTPOk, body='source body') + self.base_app.register('GET', source, HTTPOk, + headers={'x-object-meta-name': 'name'}, + body='source body') self.base_app.register('PUT', target, HTTPCreated) storlet = '/v1/AUTH_a/storlet/Storlet-1.0.jar' self.base_app.register('GET', storlet, HTTPOk, body='jar binary') @@ -416,6 +427,7 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware): put_calls = self.base_app.get_calls('PUT', target) self.assertEqual(len(put_calls), 1) self.assertEqual(put_calls[-1][3], 'source body') + self.assertEqual(put_calls[-1][2]['X-Object-Meta-Name'], 'name') self.assertNotIn('X-Run-Storlet', put_calls[-1][2]) # no invocation at proxy for debug_line in self.logger.get_log_lines('debug'):