From 965cc2fcbc18c025639c61e945bc211bf9a2783b Mon Sep 17 00:00:00 2001 From: Callum Dickinson Date: Fri, 31 Jan 2025 19:09:05 +1300 Subject: [PATCH] Add per-container storage policy to account listing Add the storage_policy attribute to the metadata returned when listing containers using the GET account API function. The storage policy of a container is a very useful attribute for telemetry and billing purposes, as it determines the location and method/redundancy of on-disk storage for the objects in the container. Ceilometer currently cannot define the storage policy as a metadata attribute in Gnocchi as GET account, the most efficient way of discovering all containers in an account, does not return the storage policy for each container. Returning the storage policy for each container in GET account is the ideal way of resolving this issue, as it allows Ceilometer to find all containers' storage policies without performing additional costly API calls. Special care has been taken to ensure the change is backwards compatible when migrating from pre-storage policy versions of Swift, even though those versions are quite old now. This special handling can be removed if support for migrating from older versions is discontinued. Closes-bug: #2097074 Change-Id: I52b37cfa49cac8675f5087bcbcfe18db0b46d887 --- swift/account/backend.py | 29 +++- swift/account/reaper.py | 3 +- swift/account/utils.py | 28 +++- swift/common/middleware/listing_formats.py | 3 + test/functional/swift_test_client.py | 3 +- test/unit/account/test_backend.py | 4 +- test/unit/account/test_reaper.py | 4 +- test/unit/account/test_server.py | 134 ++++++++++++++---- test/unit/account/test_utils.py | 82 +++++++++-- .../common/middleware/test_listing_formats.py | 82 +++++++++-- 10 files changed, 310 insertions(+), 62 deletions(-) diff --git a/swift/account/backend.py b/swift/account/backend.py index d0241a2708..dbeb1eaa0d 100644 --- a/swift/account/backend.py +++ b/swift/account/backend.py @@ -367,7 +367,7 @@ class AccountBroker(DatabaseBroker): :param allow_reserved: exclude names with reserved-byte by default :returns: list of tuples of (name, object_count, bytes_used, - put_timestamp, 0) + put_timestamp, storage_policy_index, 0) """ delim_force_gte = False if reverse: @@ -383,7 +383,8 @@ class AccountBroker(DatabaseBroker): results = [] while len(results) < limit: query = """ - SELECT name, object_count, bytes_used, put_timestamp, 0 + SELECT name, object_count, bytes_used, put_timestamp, + {storage_policy_index}, 0 FROM container WHERE """ query_args = [] @@ -415,7 +416,27 @@ class AccountBroker(DatabaseBroker): query += ' ORDER BY name %s LIMIT ?' % \ ('DESC' if reverse else '') query_args.append(limit - len(results)) - curs = conn.execute(query, query_args) + try: + # First, try querying with the storage policy index. + curs = conn.execute( + query.format( + storage_policy_index="storage_policy_index"), + query_args) + except sqlite3.OperationalError as err: + # If the storage policy column is not available, + # the database has not been migrated to the new schema + # with storage_policy_index. Re-run the query with + # storage_policy_index set to 0, which is what + # would be set once the database is migrated. + # TODO(callumdickinson): If support for migrating + # pre-storage policy versions of Swift is dropped, + # then this special handling can be removed. + if "no such column: storage_policy_index" in str(err): + curs = conn.execute( + query.format(storage_policy_index="0"), + query_args) + else: + raise curs.row_factory = None # Delimiters without a prefix is ignored, further if there @@ -452,7 +473,7 @@ class AccountBroker(DatabaseBroker): delim_force_gte = True dir_name = name[:end + len(delimiter)] if dir_name != orig_marker: - results.append([dir_name, 0, 0, '0', 1]) + results.append([dir_name, 0, 0, '0', -1, 1]) curs.close() break results.append(row) diff --git a/swift/account/reaper.py b/swift/account/reaper.py index 3323ade720..10d59ea185 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -265,7 +265,8 @@ class AccountReaper(Daemon): container_limit, '', None, None, None, allow_reserved=True)) while containers: try: - for (container, _junk, _junk, _junk, _junk) in containers: + for (container, _junk, _junk, _junk, _junk, + _junk) in containers: this_shard = ( int(md5(container.encode('utf-8'), usedforsecurity=False) diff --git a/swift/account/utils.py b/swift/account/utils.py index ac7bc3a4e9..231c207448 100644 --- a/swift/account/utils.py +++ b/swift/account/utils.py @@ -82,14 +82,34 @@ def account_listing_response(account, req, response_content_type, broker=None, prefix, delimiter, reverse, req.allow_reserved_names) data = [] - for (name, object_count, bytes_used, put_timestamp, is_subdir) \ + for (name, object_count, bytes_used, put_timestamp, + storage_policy_index, is_subdir) \ in account_list: if is_subdir: data.append({'subdir': name}) else: - data.append( - {'name': name, 'count': object_count, 'bytes': bytes_used, - 'last_modified': Timestamp(put_timestamp).isoformat}) + container = { + 'name': name, + 'count': object_count, + 'bytes': bytes_used, + 'last_modified': Timestamp(put_timestamp).isoformat} + # Add the container's storage policy to the response, unless: + # * storage_policy_index < 0, which means that + # the storage policy could not be determined + # * storage_policy_index was not found in POLICIES, + # which means the storage policy is missing from + # the Swift configuration. + # The storage policy should always be returned when + # everything is configured correctly, but clients are + # expected to be able to handle this case regardless. + if ( + storage_policy_index >= 0 + and storage_policy_index in POLICIES + ): + container['storage_policy'] = ( + POLICIES[storage_policy_index].name + ) + data.append(container) if response_content_type.endswith('/xml'): account_list = listing_formats.account_to_xml(data, account) ret = HTTPOk(body=account_list, request=req, headers=resp_headers) diff --git a/swift/common/middleware/listing_formats.py b/swift/common/middleware/listing_formats.py index 6d4ece4719..290a73152a 100644 --- a/swift/common/middleware/listing_formats.py +++ b/swift/common/middleware/listing_formats.py @@ -84,6 +84,9 @@ def account_to_xml(listing, account_name): sub = SubElement(doc, 'container') for field in ('name', 'count', 'bytes', 'last_modified'): SubElement(sub, field).text = str(record.pop(field)) + for field in ('storage_policy',): + if field in record: + SubElement(sub, field).text = str(record.pop(field)) sub.tail = '\n' return to_xml(doc) diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index 5b01b411a0..fd72c302e2 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -570,7 +570,8 @@ class Account(Base): tree = minidom.parseString(self.conn.response.read()) for x in tree.getElementsByTagName('container'): cont = {} - for key in ['name', 'count', 'bytes', 'last_modified']: + for key in ['name', 'count', 'bytes', 'last_modified', + 'storage_policy']: cont[key] = x.getElementsByTagName(key)[0].\ childNodes[0].nodeValue conts.append(cont) diff --git a/test/unit/account/test_backend.py b/test/unit/account/test_backend.py index 4cfde189c0..6c97a01159 100644 --- a/test/unit/account/test_backend.py +++ b/test/unit/account/test_backend.py @@ -1448,7 +1448,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker): # make sure we can iter containers without the migration for c in broker.list_containers_iter(1, None, None, None, None): - self.assertEqual(c, ('test_name', 1, 2, timestamp, 0)) + self.assertEqual(c, ('test_name', 1, 2, timestamp, 0, 0)) # stats table is mysteriously empty... stats = broker.get_policy_stats() @@ -1607,7 +1607,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker): # make sure "test_name" container in new database self.assertEqual(new_broker.get_info()['container_count'], 1) for c in new_broker.list_containers_iter(1, None, None, None, None): - self.assertEqual(c, ('test_name', 1, 2, timestamp, 0)) + self.assertEqual(c, ('test_name', 1, 2, timestamp, 0, 0)) # full migration successful with new_broker.get() as conn: diff --git a/test/unit/account/test_reaper.py b/test/unit/account/test_reaper.py index ae2cae5dfc..6bbc14f535 100644 --- a/test/unit/account/test_reaper.py +++ b/test/unit/account/test_reaper.py @@ -59,7 +59,7 @@ class FakeAccountBroker(object): kwargs, )) for cont in self.containers: if cont > marker: - yield cont, None, None, None, None + yield cont, None, None, None, None, None limit -= 1 if limit <= 0: break @@ -735,7 +735,7 @@ class TestReaper(unittest.TestCase): if container in self.containers_yielded: continue - yield container, None, None, None, None + yield container, None, None, None, None, None self.containers_yielded.append(container) def fake_reap_container(self, account, account_partition, diff --git a/test/unit/account/test_server.py b/test/unit/account/test_server.py index 63ee07fb69..49dc7430ed 100644 --- a/test/unit/account/test_server.py +++ b/test/unit/account/test_server.py @@ -1089,6 +1089,8 @@ class TestAccountController(unittest.TestCase): self.assertEqual(resp.content_type, 'text/plain') self.assertEqual(resp.charset, 'utf-8') + @patch_policies([StoragePolicy(0, 'zero', is_default=True), + StoragePolicy(1, 'one', is_default=False)]) def test_GET_with_containers_json(self): put_timestamps = {} req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', @@ -1108,7 +1110,8 @@ class TestAccountController(unittest.TestCase): 'X-Delete-Timestamp': '0', 'X-Object-Count': '0', 'X-Bytes-Used': '0', - 'X-Timestamp': normalize_timestamp(0)}) + 'X-Timestamp': normalize_timestamp(0), + 'X-Backend-Storage-Policy-Index': 1}) req.get_response(self.controller) req = Request.blank('/sda1/p/a?format=json', environ={'REQUEST_METHOD': 'GET'}) @@ -1117,9 +1120,11 @@ class TestAccountController(unittest.TestCase): self.assertEqual( json.loads(resp.body), [{'count': 0, 'bytes': 0, 'name': 'c1', - 'last_modified': Timestamp(put_timestamps['c1']).isoformat}, + 'last_modified': Timestamp(put_timestamps['c1']).isoformat, + 'storage_policy': POLICIES[0].name}, {'count': 0, 'bytes': 0, 'name': 'c2', - 'last_modified': Timestamp(put_timestamps['c2']).isoformat}]) + 'last_modified': Timestamp(put_timestamps['c2']).isoformat, + 'storage_policy': POLICIES[1].name}]) put_timestamps['c1'] = normalize_timestamp(3) req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': put_timestamps['c1'], @@ -1134,7 +1139,8 @@ class TestAccountController(unittest.TestCase): 'X-Delete-Timestamp': '0', 'X-Object-Count': '3', 'X-Bytes-Used': '4', - 'X-Timestamp': normalize_timestamp(0)}) + 'X-Timestamp': normalize_timestamp(0), + 'X-Backend-Storage-Policy-Index': 1}) req.get_response(self.controller) req = Request.blank('/sda1/p/a?format=json', environ={'REQUEST_METHOD': 'GET'}) @@ -1143,12 +1149,16 @@ class TestAccountController(unittest.TestCase): self.assertEqual( json.loads(resp.body), [{'count': 1, 'bytes': 2, 'name': 'c1', - 'last_modified': Timestamp(put_timestamps['c1']).isoformat}, + 'last_modified': Timestamp(put_timestamps['c1']).isoformat, + 'storage_policy': POLICIES[0].name}, {'count': 3, 'bytes': 4, 'name': 'c2', - 'last_modified': Timestamp(put_timestamps['c2']).isoformat}]) + 'last_modified': Timestamp(put_timestamps['c2']).isoformat, + 'storage_policy': POLICIES[1].name}]) self.assertEqual(resp.content_type, 'application/json') self.assertEqual(resp.charset, 'utf-8') + @patch_policies([StoragePolicy(0, 'zero', is_default=True), + StoragePolicy(1, 'one', is_default=False)]) def test_GET_with_containers_xml(self): put_timestamps = {} req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', @@ -1168,7 +1178,8 @@ class TestAccountController(unittest.TestCase): 'X-Delete-Timestamp': '0', 'X-Object-Count': '0', 'X-Bytes-Used': '0', - 'X-Timestamp': normalize_timestamp(0)}) + 'X-Timestamp': normalize_timestamp(0), + 'X-Backend-Storage-Policy-Index': 1}) req.get_response(self.controller) req = Request.blank('/sda1/p/a?format=xml', environ={'REQUEST_METHOD': 'GET'}) @@ -1183,7 +1194,8 @@ class TestAccountController(unittest.TestCase): self.assertEqual(listing[0].nodeName, 'container') container = [n for n in listing[0].childNodes if n.nodeName != '#text'] self.assertEqual(sorted([n.nodeName for n in container]), - ['bytes', 'count', 'last_modified', 'name']) + ['bytes', 'count', 'last_modified', 'name', + 'storage_policy']) node = [n for n in container if n.nodeName == 'name'][0] self.assertEqual(node.firstChild.nodeValue, 'c1') node = [n for n in container if n.nodeName == 'count'][0] @@ -1193,11 +1205,14 @@ class TestAccountController(unittest.TestCase): node = [n for n in container if n.nodeName == 'last_modified'][0] self.assertEqual(node.firstChild.nodeValue, Timestamp(put_timestamps['c1']).isoformat) + node = [n for n in container if n.nodeName == 'storage_policy'][0] + self.assertEqual(node.firstChild.nodeValue, POLICIES[0].name) self.assertEqual(listing[-1].nodeName, 'container') container = \ [n for n in listing[-1].childNodes if n.nodeName != '#text'] self.assertEqual(sorted([n.nodeName for n in container]), - ['bytes', 'count', 'last_modified', 'name']) + ['bytes', 'count', 'last_modified', 'name', + 'storage_policy']) node = [n for n in container if n.nodeName == 'name'][0] self.assertEqual(node.firstChild.nodeValue, 'c2') node = [n for n in container if n.nodeName == 'count'][0] @@ -1207,6 +1222,8 @@ class TestAccountController(unittest.TestCase): node = [n for n in container if n.nodeName == 'last_modified'][0] self.assertEqual(node.firstChild.nodeValue, Timestamp(put_timestamps['c2']).isoformat) + node = [n for n in container if n.nodeName == 'storage_policy'][0] + self.assertEqual(node.firstChild.nodeValue, POLICIES[1].name) req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '1', 'X-Delete-Timestamp': '0', @@ -1219,7 +1236,8 @@ class TestAccountController(unittest.TestCase): 'X-Delete-Timestamp': '0', 'X-Object-Count': '3', 'X-Bytes-Used': '4', - 'X-Timestamp': normalize_timestamp(0)}) + 'X-Timestamp': normalize_timestamp(0), + 'X-Backend-Storage-Policy-Index': 1}) req.get_response(self.controller) req = Request.blank('/sda1/p/a?format=xml', environ={'REQUEST_METHOD': 'GET'}) @@ -1233,7 +1251,8 @@ class TestAccountController(unittest.TestCase): self.assertEqual(listing[0].nodeName, 'container') container = [n for n in listing[0].childNodes if n.nodeName != '#text'] self.assertEqual(sorted([n.nodeName for n in container]), - ['bytes', 'count', 'last_modified', 'name']) + ['bytes', 'count', 'last_modified', 'name', + 'storage_policy']) node = [n for n in container if n.nodeName == 'name'][0] self.assertEqual(node.firstChild.nodeValue, 'c1') node = [n for n in container if n.nodeName == 'count'][0] @@ -1243,11 +1262,14 @@ class TestAccountController(unittest.TestCase): node = [n for n in container if n.nodeName == 'last_modified'][0] self.assertEqual(node.firstChild.nodeValue, Timestamp(put_timestamps['c1']).isoformat) + node = [n for n in container if n.nodeName == 'storage_policy'][0] + self.assertEqual(node.firstChild.nodeValue, POLICIES[0].name) self.assertEqual(listing[-1].nodeName, 'container') container = [ n for n in listing[-1].childNodes if n.nodeName != '#text'] self.assertEqual(sorted([n.nodeName for n in container]), - ['bytes', 'count', 'last_modified', 'name']) + ['bytes', 'count', 'last_modified', 'name', + 'storage_policy']) node = [n for n in container if n.nodeName == 'name'][0] self.assertEqual(node.firstChild.nodeValue, 'c2') node = [n for n in container if n.nodeName == 'count'][0] @@ -1257,6 +1279,8 @@ class TestAccountController(unittest.TestCase): node = [n for n in container if n.nodeName == 'last_modified'][0] self.assertEqual(node.firstChild.nodeValue, Timestamp(put_timestamps['c2']).isoformat) + node = [n for n in container if n.nodeName == 'storage_policy'][0] + self.assertEqual(node.firstChild.nodeValue, POLICIES[1].name) self.assertEqual(resp.charset, 'utf-8') def test_GET_xml_escapes_account_name(self): @@ -1347,6 +1371,8 @@ class TestAccountController(unittest.TestCase): self.assertEqual(resp.body.strip().split(b'\n'), [b'c3', b'c4']) + @patch_policies([StoragePolicy(0, 'zero', is_default=True), + StoragePolicy(1, 'one', is_default=False)]) def test_GET_limit_marker_json(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) @@ -1360,29 +1386,37 @@ class TestAccountController(unittest.TestCase): 'X-Delete-Timestamp': '0', 'X-Object-Count': '2', 'X-Bytes-Used': '3', - 'X-Timestamp': put_timestamp}) + 'X-Timestamp': put_timestamp, + 'X-Backend-Storage-Policy-Index': c % 2}) req.get_response(self.controller) req = Request.blank('/sda1/p/a?limit=3&format=json', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 200) expected = [{'count': 2, 'bytes': 3, 'name': 'c0', - 'last_modified': Timestamp('1').isoformat}, + 'last_modified': Timestamp('1').isoformat, + 'storage_policy': POLICIES[0].name}, {'count': 2, 'bytes': 3, 'name': 'c1', - 'last_modified': Timestamp('2').isoformat}, + 'last_modified': Timestamp('2').isoformat, + 'storage_policy': POLICIES[1].name}, {'count': 2, 'bytes': 3, 'name': 'c2', - 'last_modified': Timestamp('3').isoformat}] + 'last_modified': Timestamp('3').isoformat, + 'storage_policy': POLICIES[0].name}] self.assertEqual(json.loads(resp.body), expected) req = Request.blank('/sda1/p/a?limit=3&marker=c2&format=json', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 200) expected = [{'count': 2, 'bytes': 3, 'name': 'c3', - 'last_modified': Timestamp('4').isoformat}, + 'last_modified': Timestamp('4').isoformat, + 'storage_policy': POLICIES[1].name}, {'count': 2, 'bytes': 3, 'name': 'c4', - 'last_modified': Timestamp('5').isoformat}] + 'last_modified': Timestamp('5').isoformat, + 'storage_policy': POLICIES[0].name}] self.assertEqual(json.loads(resp.body), expected) + @patch_policies([StoragePolicy(0, 'zero', is_default=True), + StoragePolicy(1, 'one', is_default=False)]) def test_GET_limit_marker_xml(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) @@ -1396,7 +1430,8 @@ class TestAccountController(unittest.TestCase): 'X-Delete-Timestamp': '0', 'X-Object-Count': '2', 'X-Bytes-Used': '3', - 'X-Timestamp': put_timestamp}) + 'X-Timestamp': put_timestamp, + 'X-Backend-Storage-Policy-Index': c % 2}) req.get_response(self.controller) req = Request.blank('/sda1/p/a?limit=3&format=xml', environ={'REQUEST_METHOD': 'GET'}) @@ -1410,7 +1445,8 @@ class TestAccountController(unittest.TestCase): self.assertEqual(listing[0].nodeName, 'container') container = [n for n in listing[0].childNodes if n.nodeName != '#text'] self.assertEqual(sorted([n.nodeName for n in container]), - ['bytes', 'count', 'last_modified', 'name']) + ['bytes', 'count', 'last_modified', 'name', + 'storage_policy']) node = [n for n in container if n.nodeName == 'name'][0] self.assertEqual(node.firstChild.nodeValue, 'c0') node = [n for n in container if n.nodeName == 'count'][0] @@ -1420,11 +1456,14 @@ class TestAccountController(unittest.TestCase): node = [n for n in container if n.nodeName == 'last_modified'][0] self.assertEqual(node.firstChild.nodeValue, Timestamp('1').isoformat) + node = [n for n in container if n.nodeName == 'storage_policy'][0] + self.assertEqual(node.firstChild.nodeValue, POLICIES[0].name) self.assertEqual(listing[-1].nodeName, 'container') container = [ n for n in listing[-1].childNodes if n.nodeName != '#text'] self.assertEqual(sorted([n.nodeName for n in container]), - ['bytes', 'count', 'last_modified', 'name']) + ['bytes', 'count', 'last_modified', 'name', + 'storage_policy']) node = [n for n in container if n.nodeName == 'name'][0] self.assertEqual(node.firstChild.nodeValue, 'c2') node = [n for n in container if n.nodeName == 'count'][0] @@ -1434,6 +1473,8 @@ class TestAccountController(unittest.TestCase): node = [n for n in container if n.nodeName == 'last_modified'][0] self.assertEqual(node.firstChild.nodeValue, Timestamp('3').isoformat) + node = [n for n in container if n.nodeName == 'storage_policy'][0] + self.assertEqual(node.firstChild.nodeValue, POLICIES[0].name) req = Request.blank('/sda1/p/a?limit=3&marker=c2&format=xml', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) @@ -1446,7 +1487,8 @@ class TestAccountController(unittest.TestCase): self.assertEqual(listing[0].nodeName, 'container') container = [n for n in listing[0].childNodes if n.nodeName != '#text'] self.assertEqual(sorted([n.nodeName for n in container]), - ['bytes', 'count', 'last_modified', 'name']) + ['bytes', 'count', 'last_modified', 'name', + 'storage_policy']) node = [n for n in container if n.nodeName == 'name'][0] self.assertEqual(node.firstChild.nodeValue, 'c3') node = [n for n in container if n.nodeName == 'count'][0] @@ -1456,11 +1498,14 @@ class TestAccountController(unittest.TestCase): node = [n for n in container if n.nodeName == 'last_modified'][0] self.assertEqual(node.firstChild.nodeValue, Timestamp('4').isoformat) + node = [n for n in container if n.nodeName == 'storage_policy'][0] + self.assertEqual(node.firstChild.nodeValue, POLICIES[1].name) self.assertEqual(listing[-1].nodeName, 'container') container = [ n for n in listing[-1].childNodes if n.nodeName != '#text'] self.assertEqual(sorted([n.nodeName for n in container]), - ['bytes', 'count', 'last_modified', 'name']) + ['bytes', 'count', 'last_modified', 'name', + 'storage_policy']) node = [n for n in container if n.nodeName == 'name'][0] self.assertEqual(node.firstChild.nodeValue, 'c4') node = [n for n in container if n.nodeName == 'count'][0] @@ -1470,6 +1515,8 @@ class TestAccountController(unittest.TestCase): node = [n for n in container if n.nodeName == 'last_modified'][0] self.assertEqual(node.firstChild.nodeValue, Timestamp('5').isoformat) + node = [n for n in container if n.nodeName == 'storage_policy'][0] + self.assertEqual(node.firstChild.nodeValue, POLICIES[0].name) def test_GET_accept_wildcard(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', @@ -1911,13 +1958,18 @@ class TestAccountController(unittest.TestCase): self.assertEqual(resp.status_int // 100, 2, resp.body) for container in containers: path = '/sda1/p/%s/%s' % (account, container['name']) - req = Request.blank(path, method='PUT', headers={ + headers = { 'X-Put-Timestamp': container['timestamp'].internal, 'X-Delete-Timestamp': container.get( 'deleted', Timestamp(0)).internal, 'X-Object-Count': container['count'], 'X-Bytes-Used': container['bytes'], - }) + } + if 'storage_policy' in container: + headers['X-Backend-Storage-Policy-Index'] = ( + POLICIES.get_by_name(container['storage_policy']).idx + ) + req = Request.blank(path, method='PUT', headers=headers) resp = req.get_response(self.controller) self.assertEqual(resp.status_int // 100, 2, resp.body) @@ -1927,6 +1979,7 @@ class TestAccountController(unittest.TestCase): 'bytes': 200, 'count': 2, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[0].name, }] self._report_containers(containers) @@ -1960,17 +2013,21 @@ class TestAccountController(unittest.TestCase): self.assertEqual(json.loads(resp.body), [{ 'subdir': '%s' % get_reserved_name('null')}]) + @patch_policies([StoragePolicy(0, 'zero', is_default=True), + StoragePolicy(1, 'one', is_default=False)]) def test_delimiter_with_reserved_and_public(self): containers = [{ 'name': get_reserved_name('null', 'test01'), 'bytes': 200, 'count': 2, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[0].name, }, { 'name': 'nullish', 'bytes': 10, 'count': 10, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[1].name, }] self._report_containers(containers) @@ -2020,17 +2077,21 @@ class TestAccountController(unittest.TestCase): [{'subdir': '\x00'}] + self._expected_listing(containers)[1:]) + @patch_policies([StoragePolicy(0, 'zero', is_default=True), + StoragePolicy(1, 'one', is_default=False)]) def test_markers_with_reserved(self): containers = [{ 'name': get_reserved_name('null', 'test01'), 'bytes': 200, 'count': 2, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[0].name, }, { 'name': get_reserved_name('null', 'test02'), 'bytes': 10, 'count': 10, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[1].name, }] self._report_containers(containers) @@ -2064,6 +2125,7 @@ class TestAccountController(unittest.TestCase): 'bytes': 300, 'count': 30, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[0].name, }) self._report_containers(containers) @@ -2085,27 +2147,33 @@ class TestAccountController(unittest.TestCase): self.assertEqual(json.loads(resp.body), self._expected_listing(containers)[-1:]) + @patch_policies([StoragePolicy(0, 'zero', is_default=True), + StoragePolicy(1, 'one', is_default=False)]) def test_prefix_with_reserved(self): containers = [{ 'name': get_reserved_name('null', 'test01'), 'bytes': 200, 'count': 2, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[0].name, }, { 'name': get_reserved_name('null', 'test02'), 'bytes': 10, 'count': 10, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[1].name, }, { 'name': get_reserved_name('null', 'foo'), 'bytes': 10, 'count': 10, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[0].name, }, { 'name': get_reserved_name('nullish'), 'bytes': 300, 'count': 32, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[1].name, }] self._report_containers(containers) @@ -2125,27 +2193,33 @@ class TestAccountController(unittest.TestCase): self.assertEqual(json.loads(resp.body), self._expected_listing(containers[:2])) + @patch_policies([StoragePolicy(0, 'zero', is_default=True), + StoragePolicy(1, 'one', is_default=False)]) def test_prefix_and_delim_with_reserved(self): containers = [{ 'name': get_reserved_name('null', 'test01'), 'bytes': 200, 'count': 2, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[0].name, }, { 'name': get_reserved_name('null', 'test02'), 'bytes': 10, 'count': 10, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[1].name, }, { 'name': get_reserved_name('null', 'foo'), 'bytes': 10, 'count': 10, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[0].name, }, { 'name': get_reserved_name('nullish'), 'bytes': 300, 'count': 32, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[1].name, }] self._report_containers(containers) @@ -2166,22 +2240,27 @@ class TestAccountController(unittest.TestCase): self._expected_listing(containers[-1:]) self.assertEqual(json.loads(resp.body), expected) + @patch_policies([StoragePolicy(0, 'zero', is_default=True), + StoragePolicy(1, 'one', is_default=False)]) def test_reserved_markers_with_non_reserved(self): containers = [{ 'name': get_reserved_name('null', 'test01'), 'bytes': 200, 'count': 2, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[0].name, }, { 'name': get_reserved_name('null', 'test02'), 'bytes': 10, 'count': 10, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[1].name, }, { 'name': 'nullish', 'bytes': 300, 'count': 32, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[0].name, }] self._report_containers(containers) @@ -2221,22 +2300,27 @@ class TestAccountController(unittest.TestCase): self.assertEqual(json.loads(resp.body), self._expected_listing(containers)[1:]) + @patch_policies([StoragePolicy(0, 'zero', is_default=True), + StoragePolicy(1, 'one', is_default=False)]) def test_null_markers(self): containers = [{ 'name': get_reserved_name('null', ''), 'bytes': 200, 'count': 2, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[0].name, }, { 'name': get_reserved_name('null', 'test01'), 'bytes': 200, 'count': 2, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[1].name, }, { 'name': 'null', 'bytes': 300, 'count': 32, 'timestamp': next(self.ts), + 'storage_policy': POLICIES[0].name, }] self._report_containers(containers) diff --git a/test/unit/account/test_utils.py b/test/unit/account/test_utils.py index 473b3b053a..4ac806fb67 100644 --- a/test/unit/account/test_utils.py +++ b/test/unit/account/test_utils.py @@ -214,7 +214,58 @@ class TestAccountUtils(TestDbBase): self.assertEqual(expected, resp.headers) self.assertEqual(b'', resp.body) - @patch_policies([StoragePolicy(0, 'zero', is_default=True)]) + @patch_policies([StoragePolicy(0, 'zero', is_default=True), + StoragePolicy(1, 'one', is_default=False)]) + def test_account_listing_with_containers(self): + broker = backend.AccountBroker(self.db_path, account='a') + put_timestamp = next(self.ts) + now = time.time() + with mock.patch('time.time', new=lambda: now): + broker.initialize(put_timestamp.internal) + container_timestamp = next(self.ts) + broker.put_container('foo', + container_timestamp.internal, 0, 10, 100, 0) + broker.put_container('bar', + container_timestamp.internal, 0, 10, 100, 1) + + req = Request.blank('') + resp = utils.account_listing_response( + 'a', req, 'application/json', broker) + self.assertEqual(resp.status_int, 200) + expected = HeaderKeyDict({ + 'Content-Type': 'application/json; charset=utf-8', + 'Content-Length': 233, + 'X-Account-Container-Count': 2, + 'X-Account-Object-Count': 20, + 'X-Account-Bytes-Used': 200, + 'X-Timestamp': Timestamp(now).normal, + 'X-PUT-Timestamp': put_timestamp.normal, + 'X-Account-Storage-Policy-Zero-Container-Count': 1, + 'X-Account-Storage-Policy-Zero-Object-Count': 10, + 'X-Account-Storage-Policy-Zero-Bytes-Used': 100, + 'X-Account-Storage-Policy-One-Container-Count': 1, + 'X-Account-Storage-Policy-One-Object-Count': 10, + 'X-Account-Storage-Policy-One-Bytes-Used': 100, + }) + self.assertEqual(expected, resp.headers) + expected = [{ + "last_modified": container_timestamp.isoformat, + "count": 10, + "bytes": 100, + "name": 'foo', + 'storage_policy': POLICIES[0].name, + }, { + "last_modified": container_timestamp.isoformat, + "count": 10, + "bytes": 100, + "name": 'bar', + 'storage_policy': POLICIES[1].name, + }] + self.assertEqual(sorted(json.dumps(expected).encode('ascii')), + sorted(resp.body)) + + @patch_policies([StoragePolicy(0, 'zero', is_default=True), + StoragePolicy(1, 'one', is_default=False)]) def test_account_listing_reserved_names(self): broker = backend.AccountBroker(self.db_path, account='a') put_timestamp = next(self.ts) @@ -224,6 +275,8 @@ class TestAccountUtils(TestDbBase): container_timestamp = next(self.ts) broker.put_container(get_reserved_name('foo'), container_timestamp.internal, 0, 10, 100, 0) + broker.put_container(get_reserved_name('bar'), + container_timestamp.internal, 0, 10, 100, 1) req = Request.blank('') resp = utils.account_listing_response( @@ -232,14 +285,17 @@ class TestAccountUtils(TestDbBase): expected = HeaderKeyDict({ 'Content-Type': 'application/json; charset=utf-8', 'Content-Length': 2, - 'X-Account-Container-Count': 1, - 'X-Account-Object-Count': 10, - 'X-Account-Bytes-Used': 100, + 'X-Account-Container-Count': 2, + 'X-Account-Object-Count': 20, + 'X-Account-Bytes-Used': 200, 'X-Timestamp': Timestamp(now).normal, 'X-PUT-Timestamp': put_timestamp.normal, 'X-Account-Storage-Policy-Zero-Container-Count': 1, 'X-Account-Storage-Policy-Zero-Object-Count': 10, 'X-Account-Storage-Policy-Zero-Bytes-Used': 100, + 'X-Account-Storage-Policy-One-Container-Count': 1, + 'X-Account-Storage-Policy-One-Object-Count': 10, + 'X-Account-Storage-Policy-One-Bytes-Used': 100, }) self.assertEqual(expected, resp.headers) self.assertEqual(b'[]', resp.body) @@ -251,15 +307,18 @@ class TestAccountUtils(TestDbBase): self.assertEqual(resp.status_int, 200) expected = HeaderKeyDict({ 'Content-Type': 'application/json; charset=utf-8', - 'Content-Length': 97, - 'X-Account-Container-Count': 1, - 'X-Account-Object-Count': 10, - 'X-Account-Bytes-Used': 100, + 'Content-Length': 245, + 'X-Account-Container-Count': 2, + 'X-Account-Object-Count': 20, + 'X-Account-Bytes-Used': 200, 'X-Timestamp': Timestamp(now).normal, 'X-PUT-Timestamp': put_timestamp.normal, 'X-Account-Storage-Policy-Zero-Container-Count': 1, 'X-Account-Storage-Policy-Zero-Object-Count': 10, 'X-Account-Storage-Policy-Zero-Bytes-Used': 100, + 'X-Account-Storage-Policy-One-Container-Count': 1, + 'X-Account-Storage-Policy-One-Object-Count': 10, + 'X-Account-Storage-Policy-One-Bytes-Used': 100, }) self.assertEqual(expected, resp.headers) expected = [{ @@ -267,6 +326,13 @@ class TestAccountUtils(TestDbBase): "count": 10, "bytes": 100, "name": get_reserved_name('foo'), + 'storage_policy': POLICIES[0].name, + }, { + "last_modified": container_timestamp.isoformat, + "count": 10, + "bytes": 100, + "name": get_reserved_name('bar'), + 'storage_policy': POLICIES[1].name, }] self.assertEqual(sorted(json.dumps(expected).encode('ascii')), sorted(resp.body)) diff --git a/test/unit/common/middleware/test_listing_formats.py b/test/unit/common/middleware/test_listing_formats.py index a93d95b116..315e82de31 100644 --- a/test/unit/common/middleware/test_listing_formats.py +++ b/test/unit/common/middleware/test_listing_formats.py @@ -20,10 +20,14 @@ from swift.common.header_key_dict import HeaderKeyDict from swift.common.swob import Request, HTTPOk, HTTPNoContent from swift.common.middleware import listing_formats from swift.common.request_helpers import get_reserved_name +from swift.common.storage_policy import POLICIES from test.debug_logger import debug_logger from test.unit.common.middleware.helpers import FakeSwift +TEST_POLICIES = (POLICIES[0].name, 'Policy-1') + + class TestListingFormats(unittest.TestCase): def setUp(self): self.fake_swift = FakeSwift() @@ -32,8 +36,14 @@ class TestListingFormats(unittest.TestCase): logger=self.logger) self.fake_account_listing = json.dumps([ {'name': 'bar', 'bytes': 0, 'count': 0, - 'last_modified': '1970-01-01T00:00:00.000000'}, + 'last_modified': '1970-01-01T00:00:00.000000', + 'storage_policy': TEST_POLICIES[0]}, {'subdir': 'foo_'}, + {'name': 'foobar', 'bytes': 0, 'count': 0, + 'last_modified': '2025-01-01T00:00:00.000000', + 'storage_policy': TEST_POLICIES[1]}, + {'name': 'nobar', 'bytes': 0, 'count': 0, # Unknown policy + 'last_modified': '2025-02-01T00:00:00.000000'}, ]).encode('ascii') self.fake_container_listing = json.dumps([ {'name': 'bar', 'hash': 'etag', 'bytes': 0, @@ -44,11 +54,18 @@ class TestListingFormats(unittest.TestCase): self.fake_account_listing_with_reserved = json.dumps([ {'name': 'bar', 'bytes': 0, 'count': 0, - 'last_modified': '1970-01-01T00:00:00.000000'}, + 'last_modified': '1970-01-01T00:00:00.000000', + 'storage_policy': TEST_POLICIES[0]}, {'name': get_reserved_name('bar', 'versions'), 'bytes': 0, - 'count': 0, 'last_modified': '1970-01-01T00:00:00.000000'}, + 'count': 0, 'last_modified': '1970-01-01T00:00:00.000000', + 'storage_policy': TEST_POLICIES[0]}, {'subdir': 'foo_'}, {'subdir': get_reserved_name('foo_')}, + {'name': 'foobar', 'bytes': 0, 'count': 0, + 'last_modified': '2025-01-01T00:00:00.000000', + 'storage_policy': TEST_POLICIES[1]}, + {'name': 'nobar', 'bytes': 0, 'count': 0, # Unknown policy + 'last_modified': '2025-02-01T00:00:00.000000'}, ]).encode('ascii') self.fake_container_listing_with_reserved = json.dumps([ {'name': 'bar', 'hash': 'etag', 'bytes': 0, @@ -68,7 +85,7 @@ class TestListingFormats(unittest.TestCase): req = Request.blank('/v1/a') resp = req.get_response(self.app) - self.assertEqual(resp.body, b'bar\nfoo_\n') + self.assertEqual(resp.body, b'bar\nfoo_\nfoobar\nnobar\n') self.assertEqual(resp.headers['Content-Type'], 'text/plain; charset=utf-8') self.assertEqual(self.fake_swift.calls[-1], ( @@ -76,7 +93,7 @@ class TestListingFormats(unittest.TestCase): req = Request.blank('/v1/a?format=plain') resp = req.get_response(self.app) - self.assertEqual(resp.body, b'bar\nfoo_\n') + self.assertEqual(resp.body, b'bar\nfoo_\nfoobar\nnobar\n') self.assertEqual(resp.headers['Content-Type'], 'text/plain; charset=utf-8') self.assertEqual(self.fake_swift.calls[-1], ( @@ -98,8 +115,16 @@ class TestListingFormats(unittest.TestCase): b'', b'bar00' b'1970-01-01T00:00:00.000000' - b'', + b'%s' + b'' % TEST_POLICIES[0].encode('ascii'), b'', + b'foobar00' + b'2025-01-01T00:00:00.000000' + b'%s' + b'' % TEST_POLICIES[1].encode('ascii'), + b'nobar00' + b'2025-02-01T00:00:00.000000' + b'', b'', ]) self.assertEqual(resp.headers['Content-Type'], @@ -247,7 +272,7 @@ class TestListingFormats(unittest.TestCase): req = Request.blank('/v1/a\xe2\x98\x83') resp = req.get_response(self.app) - self.assertEqual(resp.body, b'bar\nfoo_\n') + self.assertEqual(resp.body, b'bar\nfoo_\nfoobar\nnobar\n') self.assertEqual(resp.headers['Content-Type'], 'text/plain; charset=utf-8') self.assertEqual(self.fake_swift.calls[-1], ( @@ -262,7 +287,7 @@ class TestListingFormats(unittest.TestCase): req = Request.blank('/v1/a\xe2\x98\x83', headers={ 'X-Backend-Allow-Reserved-Names': 'true'}) resp = req.get_response(self.app) - self.assertEqual(resp.body, b'bar\n%s\nfoo_\n%s\n' % ( + self.assertEqual(resp.body, b'bar\n%s\nfoo_\n%s\nfoobar\nnobar\n' % ( get_reserved_name('bar', 'versions').encode('ascii'), get_reserved_name('foo_').encode('ascii'), )) @@ -273,7 +298,7 @@ class TestListingFormats(unittest.TestCase): req = Request.blank('/v1/a\xe2\x98\x83?format=plain') resp = req.get_response(self.app) - self.assertEqual(resp.body, b'bar\nfoo_\n') + self.assertEqual(resp.body, b'bar\nfoo_\nfoobar\nnobar\n') self.assertEqual(resp.headers['Content-Type'], 'text/plain; charset=utf-8') self.assertEqual(self.fake_swift.calls[-1], ( @@ -282,7 +307,7 @@ class TestListingFormats(unittest.TestCase): req = Request.blank('/v1/a\xe2\x98\x83?format=plain', headers={ 'X-Backend-Allow-Reserved-Names': 'true'}) resp = req.get_response(self.app) - self.assertEqual(resp.body, b'bar\n%s\nfoo_\n%s\n' % ( + self.assertEqual(resp.body, b'bar\n%s\nfoo_\n%s\nfoobar\nnobar\n' % ( get_reserved_name('bar', 'versions').encode('ascii'), get_reserved_name('foo_').encode('ascii'), )) @@ -317,8 +342,16 @@ class TestListingFormats(unittest.TestCase): b'', b'bar00' b'1970-01-01T00:00:00.000000' - b'', + b'%s' + b'' % TEST_POLICIES[0].encode('ascii'), b'', + b'foobar00' + b'2025-01-01T00:00:00.000000' + b'%s' + b'' % TEST_POLICIES[1].encode('ascii'), + b'nobar00' + b'2025-02-01T00:00:00.000000' + b'', b'', ]) self.assertEqual(resp.headers['Content-Type'], @@ -334,15 +367,26 @@ class TestListingFormats(unittest.TestCase): b'', b'bar00' b'1970-01-01T00:00:00.000000' - b'', + b'%s' + b'' % TEST_POLICIES[0].encode('ascii'), b'%s' b'00' b'1970-01-01T00:00:00.000000' - b'' % get_reserved_name( - 'bar', 'versions').encode('ascii'), + b'%s' + b'' % ( + get_reserved_name('bar', 'versions').encode('ascii'), + TEST_POLICIES[0].encode('ascii'), + ), b'', b'' % get_reserved_name( 'foo_').encode('ascii'), + b'foobar00' + b'2025-01-01T00:00:00.000000' + b'%s' + b'' % TEST_POLICIES[1].encode('ascii'), + b'nobar00' + b'2025-02-01T00:00:00.000000' + b'', b'', ]) self.assertEqual(resp.headers['Content-Type'], @@ -659,8 +703,16 @@ class TestListingFormats(unittest.TestCase): body = json.dumps([ {'name': 'bar', 'hash': 'etag', 'bytes': 0, 'content_type': 'text/plain', - 'last_modified': '1970-01-01T00:00:00.000000'}, + 'last_modified': '1970-01-01T00:00:00.000000', + 'storage_policy': TEST_POLICIES[0]}, {'subdir': 'foo/'}, + {'name': 'foobar', 'hash': 'etag', 'bytes': 0, + 'content_type': 'text/plain', + 'last_modified': '2025-01-01T00:00:00.000000', + 'storage_policy': TEST_POLICIES[1]}, + {'name': 'nobar', 'hash': 'etag', 'bytes': 0, + 'content_type': 'text/plain', + 'last_modified': '2025-02-01T00:00:00.000000'}, ] * 160000).encode('ascii') self.assertGreater( # sanity len(body), listing_formats.MAX_CONTAINER_LISTING_CONTENT_LENGTH)