openstack/python-swiftclient
Code Issues Proposed changes

Correctly aggregate totals for >10k items

Browse Source 
Previously, we would write out totals for every page of listings, like

    $ swift list sync --prefix=09-21 --total -l
     80000000000
     80000000000
     80000000000
     58096000000

Now, roll those all into a single total:

    $ swift list sync --prefix=09-21 --total -l
    298096000000

Change-Id: Icc265636815220e33e8c9eec0a3ab80e9f899038
This commit is contained in:
Tim Burke
2021-09-21 16:02:47 -07:00
parent 5129b33505
commit 373fa26ce1
2 changed files with 26 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
swiftclient/shell.py
View File

@@ -531,8 +531,7 @@ Optional arguments:
def st_list(parser, args, output_manager, return_parser=False):
def _print_stats(options, stats, human):
total_count = total_bytes = 0
def _print_stats(options, stats, human, totals):
container = stats.get("container", None)
for item in stats["listing"]:
item_name = item.get('name')
@@ -543,7 +542,7 @@ def st_list(parser, args, output_manager, return_parser=False):
item_bytes = item.get('bytes')
byte_str = prt_bytes(item_bytes, human)
count = item.get('count')
total_count += count
totals['count'] += count
try:
meta = item.get('meta')
utc = gmtime(float(meta.get('x-timestamp')))
@@ -578,17 +577,7 @@ def st_list(parser, args, output_manager, return_parser=False):
output_manager.print_msg(
"%s %10s %8s %24s %s",
byte_str, date, xtime, content_type, item_name)
total_bytes += item_bytes
# report totals
if options['long'] or human:
if not container:
output_manager.print_msg(
"%12s %s", prt_bytes(total_count, True),
prt_bytes(total_bytes, human))
else:
output_manager.print_msg(
prt_bytes(total_bytes, human))
totals['bytes'] += item_bytes
parser.add_argument(
'-l', '--long', dest='long', action='store_true', default=False,
@@ -642,6 +631,7 @@ def st_list(parser, args, output_manager, return_parser=False):
try:
if not args:
stats_parts_gen = swift.list()
container = None
else:
container = args[0]
args = args[1:]
@@ -667,12 +657,24 @@ def st_list(parser, args, output_manager, return_parser=False):
sort_keys=True, indent=2)
output_manager.print_msg('')
return
totals = {'count': 0, 'bytes': 0}
for stats in stats_parts_gen:
if stats["success"]:
_print_stats(options, stats, human)
_print_stats(options, stats, human, totals)
else:
raise stats["error"]
# report totals
if options['long'] or human:
if container is None:
output_manager.print_msg(
"%12s %s", prt_bytes(totals['count'], True),
prt_bytes(totals['bytes'], human))
else:
output_manager.print_msg(
prt_bytes(totals['bytes'], human))
except SwiftError as e:
output_manager.error(e.value)

13
test/unit/test_shell.py
View File

@@ -547,9 +547,12 @@ class TestShell(unittest.TestCase):
self.assertEqual(output.out, 'object_a\n')
# Test container listing with --long
# Test container listing with --long and multiple pages
connection.return_value.get_container.side_effect = [
[None, [{'name': 'object_a', 'bytes': 0,
[None, [{'name': 'object_a', 'bytes': 3,
'content_type': 'type/content',
'last_modified': '123T456'}]],
[None, [{'name': 'object_b', 'bytes': 5,
'content_type': 'type/content',
'last_modified': '123T456'}]],
[None, []],
@@ -567,9 +570,11 @@ class TestShell(unittest.TestCase):
connection.return_value.get_container.assert_has_calls(calls)
self.assertEqual(output.out,
' 0 123 456'
' 3 123 456'
' type/content object_a\n'
' 0\n')
' 5 123 456'
' type/content object_b\n'
' 8\n')
@mock.patch('swiftclient.service.Connection')
def test_list_container_with_headers(self, connection):