From 1449532fb82b4fe3a5484b547d425dcda82df259 Mon Sep 17 00:00:00 2001 From: Kazuhiro MIYAHARA Date: Mon, 25 Dec 2017 07:09:49 +0000 Subject: [PATCH] Allow InternalClient to container/object listing with prefix This patch adds 'prefix' argument to iter_containers/iter_objects method of InternalClient. This change will be used in general task queue feature [1]. [1]: https://review.openstack.org/#/c/517389/ Change-Id: I8c2067c07fe35681fdc9403da771f451c21136d3 --- swift/common/internal_client.py | 19 ++++--- test/unit/common/test_internal_client.py | 68 ++++++++++++++++++++---- test/unit/container/test_reconciler.py | 14 ++--- 3 files changed, 79 insertions(+), 22 deletions(-) diff --git a/swift/common/internal_client.py b/swift/common/internal_client.py index 529005fe5a..89307be33e 100644 --- a/swift/common/internal_client.py +++ b/swift/common/internal_client.py @@ -241,7 +241,7 @@ class InternalClient(object): return metadata def _iter_items( - self, path, marker='', end_marker='', + self, path, marker='', end_marker='', prefix='', acceptable_statuses=(2, HTTP_NOT_FOUND)): """ Returns an iterator of items from a json listing. Assumes listing has @@ -251,6 +251,7 @@ class InternalClient(object): :param marker: Prefix of first desired item, defaults to ''. :param end_marker: Last item returned will be 'less' than this, defaults to ''. + :param prefix: Prefix of items :param acceptable_statuses: List of status for valid responses, defaults to (2, HTTP_NOT_FOUND). @@ -262,8 +263,8 @@ class InternalClient(object): while True: resp = self.make_request( - 'GET', '%s?format=json&marker=%s&end_marker=%s' % - (path, quote(marker), quote(end_marker)), + 'GET', '%s?format=json&marker=%s&end_marker=%s&prefix=%s' % + (path, quote(marker), quote(end_marker), quote(prefix)), {}, acceptable_statuses) if not resp.status_int == 200: if resp.status_int >= HTTP_MULTIPLE_CHOICES: @@ -331,7 +332,7 @@ class InternalClient(object): # account methods def iter_containers( - self, account, marker='', end_marker='', + self, account, marker='', end_marker='', prefix='', acceptable_statuses=(2, HTTP_NOT_FOUND)): """ Returns an iterator of containers dicts from an account. @@ -340,6 +341,7 @@ class InternalClient(object): :param marker: Prefix of first desired item, defaults to ''. :param end_marker: Last item returned will be 'less' than this, defaults to ''. + :param prefix: Prefix of containers :param acceptable_statuses: List of status for valid responses, defaults to (2, HTTP_NOT_FOUND). @@ -350,7 +352,8 @@ class InternalClient(object): """ path = self.make_path(account) - return self._iter_items(path, marker, end_marker, acceptable_statuses) + return self._iter_items(path, marker, end_marker, prefix, + acceptable_statuses) def get_account_info( self, account, acceptable_statuses=(2, HTTP_NOT_FOUND)): @@ -508,7 +511,7 @@ class InternalClient(object): return self._get_metadata(path, metadata_prefix, acceptable_statuses) def iter_objects( - self, account, container, marker='', end_marker='', + self, account, container, marker='', end_marker='', prefix='', acceptable_statuses=(2, HTTP_NOT_FOUND)): """ Returns an iterator of object dicts from a container. @@ -518,6 +521,7 @@ class InternalClient(object): :param marker: Prefix of first desired item, defaults to ''. :param end_marker: Last item returned will be 'less' than this, defaults to ''. + :param prefix: Prefix of objects :param acceptable_statuses: List of status for valid responses, defaults to (2, HTTP_NOT_FOUND). @@ -528,7 +532,8 @@ class InternalClient(object): """ path = self.make_path(account, container) - return self._iter_items(path, marker, end_marker, acceptable_statuses) + return self._iter_items(path, marker, end_marker, prefix, + acceptable_statuses) def set_container_metadata( self, account, container, metadata, metadata_prefix='', diff --git a/test/unit/common/test_internal_client.py b/test/unit/common/test_internal_client.py index e075d1bfc1..6c2ed582b9 100644 --- a/test/unit/common/test_internal_client.py +++ b/test/unit/common/test_internal_client.py @@ -136,19 +136,23 @@ class SetMetadataInternalClient(internal_client.InternalClient): class IterInternalClient(internal_client.InternalClient): def __init__( - self, test, path, marker, end_marker, acceptable_statuses, items): + self, test, path, marker, end_marker, prefix, acceptable_statuses, + items): self.test = test self.path = path self.marker = marker self.end_marker = end_marker + self.prefix = prefix self.acceptable_statuses = acceptable_statuses self.items = items def _iter_items( - self, path, marker='', end_marker='', acceptable_statuses=None): + self, path, marker='', end_marker='', prefix='', + acceptable_statuses=None): self.test.assertEqual(self.path, path) self.test.assertEqual(self.marker, marker) self.test.assertEqual(self.end_marker, end_marker) + self.test.assertEqual(self.prefix, prefix) self.test.assertEqual(self.acceptable_statuses, acceptable_statuses) for item in self.items: yield item @@ -667,9 +671,9 @@ class TestInternalClient(unittest.TestCase): return self.responses.pop(0) paths = [ - '/?format=json&marker=start&end_marker=end', - '/?format=json&marker=one%C3%A9&end_marker=end', - '/?format=json&marker=two&end_marker=end', + '/?format=json&marker=start&end_marker=end&prefix=', + '/?format=json&marker=one%C3%A9&end_marker=end&prefix=', + '/?format=json&marker=two&end_marker=end&prefix=', ] responses = [ @@ -685,6 +689,49 @@ class TestInternalClient(unittest.TestCase): self.assertEqual('one\xc3\xa9 two'.split(), items) + def test_iter_items_with_markers_and_prefix(self): + class Response(object): + def __init__(self, status_int, body): + self.status_int = status_int + self.body = body + + class InternalClient(internal_client.InternalClient): + def __init__(self, test, paths, responses): + self.test = test + self.paths = paths + self.responses = responses + + def make_request( + self, method, path, headers, acceptable_statuses, + body_file=None): + exp_path = self.paths.pop(0) + self.test.assertEqual(exp_path, path) + return self.responses.pop(0) + + paths = [ + '/?format=json&marker=prefixed_start&end_marker=prefixed_end' + '&prefix=prefixed_', + '/?format=json&marker=prefixed_one%C3%A9&end_marker=prefixed_end' + '&prefix=prefixed_', + '/?format=json&marker=prefixed_two&end_marker=prefixed_end' + '&prefix=prefixed_', + ] + + responses = [ + Response(200, json.dumps([{'name': 'prefixed_one\xc3\xa9'}, ])), + Response(200, json.dumps([{'name': 'prefixed_two'}, ])), + Response(204, ''), + ] + + items = [] + client = InternalClient(self, paths, responses) + for item in client._iter_items('/', marker='prefixed_start', + end_marker='prefixed_end', + prefix='prefixed_'): + items.append(item['name'].encode('utf8')) + + self.assertEqual('prefixed_one\xc3\xa9 prefixed_two'.split(), items) + def test_iter_item_read_response_if_status_is_acceptable(self): class Response(object): def __init__(self, status_int, body, app_iter): @@ -779,12 +826,13 @@ class TestInternalClient(unittest.TestCase): items = '0 1 2'.split() marker = 'some_marker' end_marker = 'some_end_marker' + prefix = 'some_prefix' acceptable_statuses = 'some_status_list' client = IterInternalClient( - self, path, marker, end_marker, acceptable_statuses, items) + self, path, marker, end_marker, prefix, acceptable_statuses, items) ret_items = [] for container in client.iter_containers( - account, marker, end_marker, + account, marker, end_marker, prefix, acceptable_statuses=acceptable_statuses): ret_items.append(container) self.assertEqual(items, ret_items) @@ -987,13 +1035,15 @@ class TestInternalClient(unittest.TestCase): path = make_path(account, container) marker = 'some_maker' end_marker = 'some_end_marker' + prefix = 'some_prefix' acceptable_statuses = 'some_status_list' items = '0 1 2'.split() client = IterInternalClient( - self, path, marker, end_marker, acceptable_statuses, items) + self, path, marker, end_marker, prefix, acceptable_statuses, items) ret_items = [] for obj in client.iter_objects( - account, container, marker, end_marker, acceptable_statuses): + account, container, marker, end_marker, prefix, + acceptable_statuses): ret_items.append(obj) self.assertEqual(items, ret_items) diff --git a/test/unit/container/test_reconciler.py b/test/unit/container/test_reconciler.py index d24f9bc507..203718729d 100644 --- a/test/unit/container/test_reconciler.py +++ b/test/unit/container/test_reconciler.py @@ -151,15 +151,17 @@ class FakeInternalClient(reconciler.InternalClient): container_listing_data.sort(key=operator.itemgetter('name')) # register container listing response container_headers = {} - container_qry_string = '?format=json&marker=&end_marker=' + container_qry_string = \ + '?format=json&marker=&end_marker=&prefix=' self.app.register('GET', container_path + container_qry_string, swob.HTTPOk, container_headers, json.dumps(container_listing_data)) if container_listing_data: obj_name = container_listing_data[-1]['name'] # client should quote and encode marker - end_qry_string = '?format=json&marker=%s&end_marker=' % ( - urllib.parse.quote(obj_name.encode('utf-8'))) + end_qry_string = \ + '?format=json&marker=%s&end_marker=&prefix=' % ( + urllib.parse.quote(obj_name.encode('utf-8'))) self.app.register('GET', container_path + end_qry_string, swob.HTTPOk, container_headers, json.dumps([])) @@ -171,11 +173,11 @@ class FakeInternalClient(reconciler.InternalClient): # register account response account_listing_data.sort(key=operator.itemgetter('name')) account_headers = {} - account_qry_string = '?format=json&marker=&end_marker=' + account_qry_string = '?format=json&marker=&end_marker=&prefix=' self.app.register('GET', account_path + account_qry_string, swob.HTTPOk, account_headers, json.dumps(account_listing_data)) - end_qry_string = '?format=json&marker=%s&end_marker=' % ( + end_qry_string = '?format=json&marker=%s&end_marker=&prefix=' % ( urllib.parse.quote(account_listing_data[-1]['name'])) self.app.register('GET', account_path + end_qry_string, swob.HTTPOk, account_headers, @@ -704,7 +706,7 @@ class TestReconcilerUtils(unittest.TestCase): def listing_qs(marker): - return "?format=json&marker=%s&end_marker=" % \ + return "?format=json&marker=%s&end_marker=&prefix=" % \ urllib.parse.quote(marker.encode('utf-8'))