openstack/swift
Code Issues Proposed changes

swift-container-info: Show shard ranges summary

Browse Source 
The current behavior is really painful when you've got hundreds of shard
ranges in a DB.  The new summary with the states is default.  Users can
add a -v/--verbose flag to see the old full detail view.

Change-Id: I0a7d65f64540f99514c52a70f9157ef060a8a892
This commit is contained in:
Tim Burke 2020-06-19 11:09:11 -07:00 committed by Clay Gerrard
parent 5cf5548a85
commit 67e3830ab9
3 changed files with 117 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bin/swift-container-info
View File

@ -42,6 +42,10 @@ if __name__ == '__main__':
parser.add_option(
'--drop-prefixes', default=False, action="store_true",
help="When outputting metadata, drop the per-section common prefixes")
parser.add_option(
'-v', '--verbose', default=False, action="store_true",
help="Show all shard ranges. By default, only the number of shard "
"ranges is displayed if there are many shards.")
options, args = parser.parse_args()

46
swift/cli/info.py
View File

@ -16,6 +16,7 @@ import json
import os
import sqlite3
from hashlib import md5
from collections import defaultdict
from six.moves import urllib
@ -195,7 +196,8 @@ def print_ring_locations(ring, datadir, account, container=None, obj=None,
'real value is set in the config file on each storage node.')
def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False):
def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False,
verbose=False):
"""
print out data base info/metadata based on its type
@ -309,20 +311,31 @@ def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False):
print(' Type: %s' % shard_type)
print(' State: %s' % info['db_state'])
if info.get('shard_ranges'):
print('Shard Ranges (%d):' % len(info['shard_ranges']))
num_shards = len(info['shard_ranges'])
print('Shard Ranges (%d):' % num_shards)
count_by_state = defaultdict(int)
for srange in info['shard_ranges']:
srange = dict(srange, state_text=srange.state_text)
print(' Name: %(name)s' % srange)
print(' lower: %(lower)r, upper: %(upper)r' % srange)
print(' Object Count: %(object_count)d, Bytes Used: '
'%(bytes_used)d, State: %(state_text)s (%(state)d)'
% srange)
print(' Created at: %s (%s)'
% (Timestamp(srange['timestamp']).isoformat,
srange['timestamp']))
print(' Meta Timestamp: %s (%s)'
% (Timestamp(srange['meta_timestamp']).isoformat,
srange['meta_timestamp']))
count_by_state[(srange.state, srange.state_text)] += 1
print(' States:')
for key_state, count in sorted(count_by_state.items()):
key, state = key_state
print(' %9s: %s' % (state, count))
if verbose:
for srange in info['shard_ranges']:
srange = dict(srange, state_text=srange.state_text)
print(' Name: %(name)s' % srange)
print(' lower: %(lower)r, upper: %(upper)r' % srange)
print(' Object Count: %(object_count)d, Bytes Used: '
'%(bytes_used)d, State: %(state_text)s (%(state)d)'
% srange)
print(' Created at: %s (%s)'
% (Timestamp(srange['timestamp']).isoformat,
srange['timestamp']))
print(' Meta Timestamp: %s (%s)'
% (Timestamp(srange['meta_timestamp']).isoformat,
srange['meta_timestamp']))
else:
print('(Use -v/--verbose to show more Shard Ranges details)')
def print_obj_metadata(metadata, drop_prefixes=False):
@ -420,7 +433,7 @@ def print_obj_metadata(metadata, drop_prefixes=False):
def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False,
drop_prefixes=False):
drop_prefixes=False, verbose=False):
if db_type not in ('account', 'container'):
print("Unrecognized DB type: internal error")
raise InfoSystemExit()
@ -452,7 +465,8 @@ def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False,
sranges = broker.get_shard_ranges()
if sranges:
info['shard_ranges'] = sranges
print_db_info_metadata(db_type, info, broker.metadata, drop_prefixes)
print_db_info_metadata(
db_type, info, broker.metadata, drop_prefixes, verbose)
try:
ring = Ring(swift_dir, ring_name=db_type)
except Exception:

89
test/unit/cli/test_info.py
View File

@ -141,8 +141,8 @@ Metadata:
No system metadata found in db file
User Metadata: {'x-account-meta-mydata': 'swift'}'''
self.assertEqual(sorted(out.getvalue().strip().split('\n')),
sorted(exp_out.split('\n')))
self.assertEqual(out.getvalue().strip().split('\n'),
exp_out.split('\n'))
info = dict(
account='acct',
@ -269,7 +269,7 @@ No system metadata found in db file
id='abadf100d0ddba11')
out = StringIO()
with mock.patch('sys.stdout', out):
print_db_info_metadata('container', info, {})
print_db_info_metadata('container', info, {}, verbose=True)
exp_out = '''Path: /acct/cont
Account: acct
Container: cont
@ -295,6 +295,10 @@ Sharding Metadata:
Type: root
State: sharded
Shard Ranges (3):
States:
found: 1
created: 1
cleaved: 1
Name: .sharded_a/shard_range_1
lower: '1a', upper: '1z'
Object Count: 1, Bytes Used: 1, State: cleaved (30)
@ -311,8 +315,77 @@ Shard Ranges (3):
Created at: 1970-01-01T00:00:03.000000 (0000000003.00000)
Meta Timestamp: 1970-01-01T00:00:03.000000 (0000000003.00000)''' %\
POLICIES[0].name
self.assertEqual(sorted(out.getvalue().strip().split('\n')),
sorted(exp_out.strip().split('\n')))
self.assertEqual(out.getvalue().strip().split('\n'),
exp_out.strip().split('\n'))
def test_print_db_info_metadata_with_many_shard_ranges(self):
shard_ranges = [utils.ShardRange(
name='.sharded_a/shard_range_%s' % i,
timestamp=utils.Timestamp(i), lower='%02da' % i,
upper='%02dz' % i, object_count=i, bytes_used=i,
meta_timestamp=utils.Timestamp(i)) for i in range(1, 20)]
shard_ranges[0].state = utils.ShardRange.CLEAVED
shard_ranges[1].state = utils.ShardRange.CREATED
info = dict(
account='acct',
container='cont',
storage_policy_index=0,
created_at='0000000100.10000',
put_timestamp='0000000106.30000',
delete_timestamp='0000000107.90000',
status_changed_at='0000000108.30000',
object_count='20',
bytes_used='42',
reported_put_timestamp='0000010106.30000',
reported_delete_timestamp='0000010107.90000',
reported_object_count='20',
reported_bytes_used='42',
db_state=SHARDED,
is_root=True,
shard_ranges=shard_ranges,
is_deleted=False,
hash='abaddeadbeefcafe',
id='abadf100d0ddba11')
out = StringIO()
with mock.patch('sys.stdout', out):
print_db_info_metadata('container', info, {})
exp_out = '''
Path: /acct/cont
Account: acct
Container: cont
Deleted: False
Container Hash: d49d0ecbb53be1fcc49624f2f7c7ccae
Metadata:
Created at: 1970-01-01T00:01:40.100000 (0000000100.10000)
Put Timestamp: 1970-01-01T00:01:46.300000 (0000000106.30000)
Delete Timestamp: 1970-01-01T00:01:47.900000 (0000000107.90000)
Status Timestamp: 1970-01-01T00:01:48.300000 (0000000108.30000)
Object Count: 20
Bytes Used: 42
Storage Policy: %s (0)
Reported Put Timestamp: 1970-01-01T02:48:26.300000 (0000010106.30000)
Reported Delete Timestamp: 1970-01-01T02:48:27.900000 (0000010107.90000)
Reported Object Count: 20
Reported Bytes Used: 42
Chexor: abaddeadbeefcafe
UUID: abadf100d0ddba11
No system metadata found in db file
No user metadata found in db file
Sharding Metadata:
Type: root
State: sharded
Shard Ranges (19):
States:
found: 17
created: 1
cleaved: 1
(Use -v/--verbose to show more Shard Ranges details)
''' %\
POLICIES[0].name
self.assertEqual(out.getvalue().strip().split('\n'),
exp_out.strip().split('\n'))
def test_print_db_info_metadata_with_shard_ranges_bis(self):
@ -346,7 +419,7 @@ Shard Ranges (3):
info['is_deleted'] = False
out = StringIO()
with mock.patch('sys.stdout', out):
print_db_info_metadata('container', info, {})
print_db_info_metadata('container', info, {}, verbose=True)
if six.PY2:
s_a = '\\xe3\\x82\\xa2'
s_ya = '\\xe3\\x83\\xa4'
@ -378,6 +451,10 @@ Sharding Metadata:
Type: root
State: sharded
Shard Ranges (3):
States:
found: 1
created: 1
cleaved: 1
Name: .sharded_a/shard_range_1
lower: '1%s', upper: '1%s'
Object Count: 1, Bytes Used: 1, State: cleaved (30)