From d0367fdf1909ad0613e094473af23b61f8b6d34f Mon Sep 17 00:00:00 2001 From: gholt Date: Sun, 5 Sep 2010 21:06:16 -0700 Subject: [PATCH] Updated direct_client to match the changes in client --- bin/swift-stats-report | 2 +- swift/account/reaper.py | 2 +- swift/common/client.py | 8 --- swift/common/direct_client.py | 71 +++++++++++++++----------- test/probe/test_object_async_update.py | 4 +- test/probe/test_object_handoff.py | 22 ++++---- test/unit/common/test_direct_client.py | 1 - 7 files changed, 55 insertions(+), 55 deletions(-) diff --git a/bin/swift-stats-report b/bin/swift-stats-report index bbb4410d2b..3f735877cf 100755 --- a/bin/swift-stats-report +++ b/bin/swift-stats-report @@ -86,7 +86,7 @@ def audit(coropool, connpool, account, container_ring, object_ring, options): retries_done[0] += attempts - 1 found = True if not estimated_objects: - estimated_objects = info[0] + estimated_objects = int(info['x-container-object-count']) except ClientException, err: if err.http_status not in (404, 507): error_log('Giving up on /%s/%s/%s: %s' % (part, account, diff --git a/swift/account/reaper.py b/swift/account/reaper.py index 02f1ab76ad..87fa026ebd 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -310,7 +310,7 @@ class AccountReaper(Daemon): try: objects = direct_get_container(node, part, account, container, marker=marker, conn_timeout=self.conn_timeout, - response_timeout=self.node_timeout) + response_timeout=self.node_timeout)[1] self.stats_return_codes[2] = \ self.stats_return_codes.get(2, 0) + 1 except ClientException, err: diff --git a/swift/common/client.py b/swift/common/client.py index 0448442526..e17afee958 100644 --- a/swift/common/client.py +++ b/swift/common/client.py @@ -504,10 +504,6 @@ def get_object(url, token, container, name, http_conn=None, raise ClientException('Object GET failed', http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=path, http_status=resp.status, http_reason=resp.reason) - metadata = {} - for key, value in resp.getheaders(): - if key.lower().startswith('x-object-meta-'): - metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) if resp_chunk_size: def _object_body(): @@ -550,10 +546,6 @@ def head_object(url, token, container, name, http_conn=None): raise ClientException('Object HEAD failed', http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=path, http_status=resp.status, http_reason=resp.reason) - metadata = {} - for key, value in resp.getheaders(): - if key.lower().startswith('x-object-meta-'): - metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) resp_headers = {} for header, value in resp.getheaders(): resp_headers[header.lower()] = value diff --git a/swift/common/direct_client.py b/swift/common/direct_client.py index 1d7030c09d..71fe3005a4 100644 --- a/swift/common/direct_client.py +++ b/swift/common/direct_client.py @@ -47,7 +47,8 @@ def direct_head_container(node, part, account, container, conn_timeout=5, :param container: container name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response - :returns: tuple of (object count, bytes used) + :returns: a dict containing the response's headers (all header names will + be lowercase) """ path = '/%s/%s' % (account, container) with Timeout(conn_timeout): @@ -65,8 +66,10 @@ def direct_head_container(node, part, account, container, conn_timeout=5, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) - return int(resp.getheader('x-container-object-count')), \ - int(resp.getheader('x-container-bytes-used')) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers def direct_get_container(node, part, account, container, marker=None, @@ -85,7 +88,8 @@ def direct_get_container(node, part, account, container, marker=None, :param delimeter: delimeter for the query :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response - :returns: list of objects + :returns: a tuple of (response headers, a list of objects) The response + headers will be a dict and all header names will be lowercase. """ path = '/%s/%s' % (account, container) qs = 'format=json' @@ -111,10 +115,13 @@ def direct_get_container(node, part, account, container, marker=None, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value if resp.status == 204: resp.read() - return [] - return json_loads(resp.read()) + return resp_headers, [] + return resp_headers, json_loads(resp.read()) def direct_delete_container(node, part, account, container, conn_timeout=5, @@ -126,6 +133,7 @@ def direct_delete_container(node, part, account, container, conn_timeout=5, 'DELETE', path, headers) with Timeout(response_timeout): resp = conn.getresponse() + resp.read() if resp.status < 200 or resp.status >= 300: raise ClientException( 'Container server %s:%s direct DELETE %s gave status %s' % @@ -135,7 +143,6 @@ def direct_delete_container(node, part, account, container, conn_timeout=5, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) - return resp def direct_head_object(node, part, account, container, obj, conn_timeout=5, @@ -150,8 +157,8 @@ def direct_head_object(node, part, account, container, obj, conn_timeout=5, :param obj: object name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response - :returns: tuple of (content-type, object size, last modified timestamp, - etag, metadata dictionary) + :returns: a dict containing the response's headers (all header names will + be lowercase) """ path = '/%s/%s/%s' % (account, container, obj) with Timeout(conn_timeout): @@ -169,19 +176,14 @@ def direct_head_object(node, part, account, container, obj, conn_timeout=5, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) - metadata = {} - for key, value in resp.getheaders(): - if key.lower().startswith('x-object-meta-'): - metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) - return resp.getheader('content-type'), \ - int(resp.getheader('content-length')), \ - resp.getheader('last-modified'), \ - resp.getheader('etag').strip('"'), \ - metadata + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers def direct_get_object(node, part, account, container, obj, conn_timeout=5, - response_timeout=15): + response_timeout=15, resp_chunk_size=None): """ Get object directly from the object server. @@ -192,7 +194,9 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5, :param obj: object name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response - :returns: object + :param resp_chunk_size: if defined, chunk size of data to read. + :returns: a tuple of (response headers, the object's contents) The response + headers will be a dict and all header names will be lowercase. """ path = '/%s/%s/%s' % (account, container, obj) with Timeout(conn_timeout): @@ -201,6 +205,7 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5, with Timeout(response_timeout): resp = conn.getresponse() if resp.status < 200 or resp.status >= 300: + resp.read() raise ClientException( 'Object server %s:%s direct GET %s gave status %s' % (node['ip'], node['port'], @@ -209,16 +214,20 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) - metadata = {} - for key, value in resp.getheaders(): - if key.lower().startswith('x-object-meta-'): - metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) - return (resp.getheader('content-type'), - int(resp.getheader('content-length')), - resp.getheader('last-modified'), - resp.getheader('etag').strip('"'), - metadata, - resp.read()) + if resp_chunk_size: + + def _object_body(): + buf = resp.read(resp_chunk_size) + while buf: + yield buf + buf = resp.read(resp_chunk_size) + object_body = _object_body() + else: + object_body = resp.read() + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers, object_body def direct_delete_object(node, part, account, container, obj, @@ -242,6 +251,7 @@ def direct_delete_object(node, part, account, container, obj, 'DELETE', path, headers) with Timeout(response_timeout): resp = conn.getresponse() + resp.read() if resp.status < 200 or resp.status >= 300: raise ClientException( 'Object server %s:%s direct DELETE %s gave status %s' % @@ -251,7 +261,6 @@ def direct_delete_object(node, part, account, container, obj, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) - return resp def retry(func, *args, **kwargs): diff --git a/test/probe/test_object_async_update.py b/test/probe/test_object_async_update.py index 4b4c3a49b7..a5d5852c68 100755 --- a/test/probe/test_object_async_update.py +++ b/test/probe/test_object_async_update.py @@ -52,7 +52,7 @@ class TestObjectAsyncUpdate(unittest.TestCase): ((cnode['port'] - 6001) / 10)]).pid sleep(2) self.assert_(not direct_client.direct_get_container(cnode, cpart, - self.account, container)) + self.account, container)[1]) ps = [] for n in xrange(1, 5): ps.append(Popen(['swift-object-updater', @@ -60,7 +60,7 @@ class TestObjectAsyncUpdate(unittest.TestCase): for p in ps: p.wait() objs = [o['name'] for o in direct_client.direct_get_container(cnode, - cpart, self.account, container)] + cpart, self.account, container)[1]] self.assert_(obj in objs) diff --git a/test/probe/test_object_handoff.py b/test/probe/test_object_handoff.py index 37767cbb5e..006f0d3a1e 100755 --- a/test/probe/test_object_handoff.py +++ b/test/probe/test_object_handoff.py @@ -81,7 +81,7 @@ class TestObjectHandoff(unittest.TestCase): for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container(cnode, cpart, - self.account, container)] + self.account, container)[1]] if obj not in objs: raise Exception( 'Container server %s:%s did not know about object' % @@ -127,9 +127,9 @@ class TestObjectHandoff(unittest.TestCase): kill(self.pids[self.port2server[onode['port']]], SIGTERM) client.post_object(self.url, self.token, container, obj, headers={'x-object-meta-probe': 'value'}) - ometadata = client.head_object(self.url, self.token, container, obj) - if ometadata.get('x-object-meta-probe') != 'value': - raise Exception('Metadata incorrect, was %s' % repr(ometadata)) + oheaders = client.head_object(self.url, self.token, container, obj) + if oheaders.get('x-object-meta-probe') != 'value': + raise Exception('Metadata incorrect, was %s' % repr(oheaders)) exc = False try: direct_client.direct_get_object(another_onode, opart, self.account, @@ -144,9 +144,9 @@ class TestObjectHandoff(unittest.TestCase): '/etc/swift/object-server/%d.conf' % ((onode['port'] - 6000) / 10)]).pid sleep(2) - ometadata = direct_client.direct_get_object(onode, opart, self.account, - container, obj)[-2] - if ometadata.get('probe') == 'value': + oheaders = direct_client.direct_get_object(onode, opart, self.account, + container, obj)[0] + if oheaders.get('x-object-meta-probe') == 'value': raise Exception('Previously downed object server had the new ' 'metadata when it should not have it') # Run the extra server last so it'll remove it's extra partition @@ -160,9 +160,9 @@ class TestObjectHandoff(unittest.TestCase): call(['swift-object-replicator', '/etc/swift/object-server/%d.conf' % ((another_onode['port'] - 6000) / 10), 'once']) - ometadata = direct_client.direct_get_object(onode, opart, self.account, - container, obj)[-2] - if ometadata.get('probe') != 'value': + oheaders = direct_client.direct_get_object(onode, opart, self.account, + container, obj)[0] + if oheaders.get('x-object-meta-probe') != 'value': raise Exception( 'Previously downed object server did not have the new metadata') @@ -182,7 +182,7 @@ class TestObjectHandoff(unittest.TestCase): for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( - cnode, cpart, self.account, container)] + cnode, cpart, self.account, container)[1]] if obj in objs: raise Exception( 'Container server %s:%s still knew about object' % diff --git a/test/unit/common/test_direct_client.py b/test/unit/common/test_direct_client.py index 029791289e..a925c118bb 100644 --- a/test/unit/common/test_direct_client.py +++ b/test/unit/common/test_direct_client.py @@ -16,7 +16,6 @@ # TODO: Tests import unittest -from swift.common import direct_client class TestAuditor(unittest.TestCase):