openstack/python-swiftclient
Code Issues Proposed changes

Correctly aggregate totals for >10k items

Browse Source 
Previously, we would write out totals for every page of listings, like

    $ swift list sync --prefix=09-21 --total -l
     80000000000
     80000000000
     80000000000
     58096000000

Now, roll those all into a single total:

    $ swift list sync --prefix=09-21 --total -l
    298096000000

Change-Id: Icc265636815220e33e8c9eec0a3ab80e9f899038
This commit is contained in:
Tim Burke 2021-09-21 16:02:47 -07:00
parent 5129b33505
commit 373fa26ce1
2 changed files with 26 additions and 19 deletions
swiftclient
shell.py
test/unit
test_shell.py

32
swiftclient/shell.py

@ -531,8 +531,7 @@ Optional arguments:
def st_list(parser, args, output_manager, return_parser=False): def st_list(parser, args, output_manager, return_parser=False):
def _print_stats(options, stats, human): def _print_stats(options, stats, human, totals):
total_count = total_bytes = 0
container = stats.get("container", None) container = stats.get("container", None)
for item in stats["listing"]: for item in stats["listing"]:
item_name = item.get('name') item_name = item.get('name')
@ -543,7 +542,7 @@ def st_list(parser, args, output_manager, return_parser=False):
item_bytes = item.get('bytes') item_bytes = item.get('bytes')
byte_str = prt_bytes(item_bytes, human) byte_str = prt_bytes(item_bytes, human)
count = item.get('count') count = item.get('count')
total_count += count totals['count'] += count
try: try:
meta = item.get('meta') meta = item.get('meta')
utc = gmtime(float(meta.get('x-timestamp'))) utc = gmtime(float(meta.get('x-timestamp')))
@ -578,17 +577,7 @@ def st_list(parser, args, output_manager, return_parser=False):
output_manager.print_msg( output_manager.print_msg(
"%s %10s %8s %24s %s", "%s %10s %8s %24s %s",
byte_str, date, xtime, content_type, item_name) byte_str, date, xtime, content_type, item_name)
total_bytes += item_bytes totals['bytes'] += item_bytes
# report totals
if options['long'] or human:
if not container:
output_manager.print_msg(
"%12s %s", prt_bytes(total_count, True),
prt_bytes(total_bytes, human))
else:
output_manager.print_msg(
prt_bytes(total_bytes, human))
parser.add_argument( parser.add_argument(
'-l', '--long', dest='long', action='store_true', default=False, '-l', '--long', dest='long', action='store_true', default=False,
@ -642,6 +631,7 @@ def st_list(parser, args, output_manager, return_parser=False):
try: try:
if not args: if not args:
stats_parts_gen = swift.list() stats_parts_gen = swift.list()
container = None
else: else:
container = args[0] container = args[0]
args = args[1:] args = args[1:]
@ -667,12 +657,24 @@ def st_list(parser, args, output_manager, return_parser=False):
sort_keys=True, indent=2) sort_keys=True, indent=2)
output_manager.print_msg('') output_manager.print_msg('')
return return
totals = {'count': 0, 'bytes': 0}
for stats in stats_parts_gen: for stats in stats_parts_gen:
if stats["success"]: if stats["success"]:
_print_stats(options, stats, human) _print_stats(options, stats, human, totals)
else: else:
raise stats["error"] raise stats["error"]
# report totals
if options['long'] or human:
if container is None:
output_manager.print_msg(
"%12s %s", prt_bytes(totals['count'], True),
prt_bytes(totals['bytes'], human))
else:
output_manager.print_msg(
prt_bytes(totals['bytes'], human))
except SwiftError as e: except SwiftError as e:
output_manager.error(e.value) output_manager.error(e.value)

13
test/unit/test_shell.py

@ -547,9 +547,12 @@ class TestShell(unittest.TestCase):
self.assertEqual(output.out, 'object_a\n') self.assertEqual(output.out, 'object_a\n')
# Test container listing with --long # Test container listing with --long and multiple pages
connection.return_value.get_container.side_effect = [ connection.return_value.get_container.side_effect = [
[None, [{'name': 'object_a', 'bytes': 0, [None, [{'name': 'object_a', 'bytes': 3,
'content_type': 'type/content',
'last_modified': '123T456'}]],
[None, [{'name': 'object_b', 'bytes': 5,
'content_type': 'type/content', 'content_type': 'type/content',
'last_modified': '123T456'}]], 'last_modified': '123T456'}]],
[None, []], [None, []],
@ -567,9 +570,11 @@ class TestShell(unittest.TestCase):
connection.return_value.get_container.assert_has_calls(calls) connection.return_value.get_container.assert_has_calls(calls)
self.assertEqual(output.out, self.assertEqual(output.out,
' 0 123 456' ' 3 123 456'
' type/content object_a\n' ' type/content object_a\n'
' 0\n') ' 5 123 456'
' type/content object_b\n'
' 8\n')
@mock.patch('swiftclient.service.Connection') @mock.patch('swiftclient.service.Connection')
def test_list_container_with_headers(self, connection): def test_list_container_with_headers(self, connection):