swift-container-info: Show shard ranges summary

The current behavior is really painful when you've got hundreds of shard
ranges in a DB.  The new summary with the states is default.  Users can
add a -v/--verbose flag to see the old full detail view.

Change-Id: I0a7d65f64540f99514c52a70f9157ef060a8a892

bin/swift-container-info

@@ -42,6 +42,10 @@ if __name__ == '__main__':
     parser.add_option(
         '--drop-prefixes', default=False, action="store_true",
         help="When outputting metadata, drop the per-section common prefixes")
+    parser.add_option(
+        '-v', '--verbose', default=False, action="store_true",
+        help="Show all shard ranges. By default, only the number of shard "
+             "ranges is displayed if there are many shards.")
 
     options, args = parser.parse_args()
 

swift/cli/info.py

@@ -16,6 +16,7 @@ import json
 import os
 import sqlite3
 from hashlib import md5
+from collections import defaultdict
 
 from six.moves import urllib
 
@@ -195,7 +196,8 @@ def print_ring_locations(ring, datadir, account, container=None, obj=None,
           'real value is set in the config file on each storage node.')
 
 
-def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False):
+def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False,
+                           verbose=False):
     """
     print out data base info/metadata based on its type
 
@@ -309,20 +311,31 @@ def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False):
         print('  Type: %s' % shard_type)
         print('  State: %s' % info['db_state'])
     if info.get('shard_ranges'):
-        print('Shard Ranges (%d):' % len(info['shard_ranges']))
+        num_shards = len(info['shard_ranges'])
+        print('Shard Ranges (%d):' % num_shards)
+        count_by_state = defaultdict(int)
         for srange in info['shard_ranges']:
-            srange = dict(srange, state_text=srange.state_text)
+            count_by_state[(srange.state, srange.state_text)] += 1
-            print('  Name: %(name)s' % srange)
+        print('  States:')
-            print('    lower: %(lower)r, upper: %(upper)r' % srange)
+        for key_state, count in sorted(count_by_state.items()):
-            print('    Object Count: %(object_count)d, Bytes Used: '
+            key, state = key_state
-                  '%(bytes_used)d, State: %(state_text)s (%(state)d)'
+            print('    %9s: %s' % (state, count))
-                  % srange)
+        if verbose:
-            print('    Created at: %s (%s)'
+            for srange in info['shard_ranges']:
-                  % (Timestamp(srange['timestamp']).isoformat,
+                srange = dict(srange, state_text=srange.state_text)
-                     srange['timestamp']))
+                print('  Name: %(name)s' % srange)
-            print('    Meta Timestamp: %s (%s)'
+                print('    lower: %(lower)r, upper: %(upper)r' % srange)
-                  % (Timestamp(srange['meta_timestamp']).isoformat,
+                print('    Object Count: %(object_count)d, Bytes Used: '
-                     srange['meta_timestamp']))
+                      '%(bytes_used)d, State: %(state_text)s (%(state)d)'
+                      % srange)
+                print('    Created at: %s (%s)'
+                      % (Timestamp(srange['timestamp']).isoformat,
+                         srange['timestamp']))
+                print('    Meta Timestamp: %s (%s)'
+                      % (Timestamp(srange['meta_timestamp']).isoformat,
+                         srange['meta_timestamp']))
+        else:
+            print('(Use -v/--verbose to show more Shard Ranges details)')
 
 
 def print_obj_metadata(metadata, drop_prefixes=False):
@@ -420,7 +433,7 @@ def print_obj_metadata(metadata, drop_prefixes=False):
 
 
 def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False,
-               drop_prefixes=False):
+               drop_prefixes=False, verbose=False):
     if db_type not in ('account', 'container'):
         print("Unrecognized DB type: internal error")
         raise InfoSystemExit()
@@ -452,7 +465,8 @@ def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False,
         sranges = broker.get_shard_ranges()
         if sranges:
             info['shard_ranges'] = sranges
-    print_db_info_metadata(db_type, info, broker.metadata, drop_prefixes)
+    print_db_info_metadata(
+        db_type, info, broker.metadata, drop_prefixes, verbose)
     try:
         ring = Ring(swift_dir, ring_name=db_type)
     except Exception:

test/unit/cli/test_info.py

@@ -141,8 +141,8 @@ Metadata:
 No system metadata found in db file
   User Metadata: {'x-account-meta-mydata': 'swift'}'''
 
-        self.assertEqual(sorted(out.getvalue().strip().split('\n')),
+        self.assertEqual(out.getvalue().strip().split('\n'),
-                         sorted(exp_out.split('\n')))
+                         exp_out.split('\n'))
 
         info = dict(
             account='acct',
@@ -269,7 +269,7 @@ No system metadata found in db file
             id='abadf100d0ddba11')
         out = StringIO()
         with mock.patch('sys.stdout', out):
-            print_db_info_metadata('container', info, {})
+            print_db_info_metadata('container', info, {}, verbose=True)
         exp_out = '''Path: /acct/cont
   Account: acct
   Container: cont
@@ -295,6 +295,10 @@ Sharding Metadata:
   Type: root
   State: sharded
 Shard Ranges (3):
+  States:
+        found: 1
+      created: 1
+      cleaved: 1
   Name: .sharded_a/shard_range_1
     lower: '1a', upper: '1z'
     Object Count: 1, Bytes Used: 1, State: cleaved (30)
@@ -311,8 +315,77 @@ Shard Ranges (3):
     Created at: 1970-01-01T00:00:03.000000 (0000000003.00000)
     Meta Timestamp: 1970-01-01T00:00:03.000000 (0000000003.00000)''' %\
                   POLICIES[0].name
-        self.assertEqual(sorted(out.getvalue().strip().split('\n')),
+        self.assertEqual(out.getvalue().strip().split('\n'),
-                         sorted(exp_out.strip().split('\n')))
+                         exp_out.strip().split('\n'))
+
+    def test_print_db_info_metadata_with_many_shard_ranges(self):
+
+        shard_ranges = [utils.ShardRange(
+            name='.sharded_a/shard_range_%s' % i,
+            timestamp=utils.Timestamp(i), lower='%02da' % i,
+            upper='%02dz' % i, object_count=i, bytes_used=i,
+            meta_timestamp=utils.Timestamp(i)) for i in range(1, 20)]
+        shard_ranges[0].state = utils.ShardRange.CLEAVED
+        shard_ranges[1].state = utils.ShardRange.CREATED
+
+        info = dict(
+            account='acct',
+            container='cont',
+            storage_policy_index=0,
+            created_at='0000000100.10000',
+            put_timestamp='0000000106.30000',
+            delete_timestamp='0000000107.90000',
+            status_changed_at='0000000108.30000',
+            object_count='20',
+            bytes_used='42',
+            reported_put_timestamp='0000010106.30000',
+            reported_delete_timestamp='0000010107.90000',
+            reported_object_count='20',
+            reported_bytes_used='42',
+            db_state=SHARDED,
+            is_root=True,
+            shard_ranges=shard_ranges,
+            is_deleted=False,
+            hash='abaddeadbeefcafe',
+            id='abadf100d0ddba11')
+        out = StringIO()
+        with mock.patch('sys.stdout', out):
+            print_db_info_metadata('container', info, {})
+        exp_out = '''
+Path: /acct/cont
+  Account: acct
+  Container: cont
+  Deleted: False
+  Container Hash: d49d0ecbb53be1fcc49624f2f7c7ccae
+Metadata:
+  Created at: 1970-01-01T00:01:40.100000 (0000000100.10000)
+  Put Timestamp: 1970-01-01T00:01:46.300000 (0000000106.30000)
+  Delete Timestamp: 1970-01-01T00:01:47.900000 (0000000107.90000)
+  Status Timestamp: 1970-01-01T00:01:48.300000 (0000000108.30000)
+  Object Count: 20
+  Bytes Used: 42
+  Storage Policy: %s (0)
+  Reported Put Timestamp: 1970-01-01T02:48:26.300000 (0000010106.30000)
+  Reported Delete Timestamp: 1970-01-01T02:48:27.900000 (0000010107.90000)
+  Reported Object Count: 20
+  Reported Bytes Used: 42
+  Chexor: abaddeadbeefcafe
+  UUID: abadf100d0ddba11
+No system metadata found in db file
+No user metadata found in db file
+Sharding Metadata:
+  Type: root
+  State: sharded
+Shard Ranges (19):
+  States:
+        found: 17
+      created: 1
+      cleaved: 1
+(Use -v/--verbose to show more Shard Ranges details)
+''' %\
+                  POLICIES[0].name
+        self.assertEqual(out.getvalue().strip().split('\n'),
+                         exp_out.strip().split('\n'))
 
     def test_print_db_info_metadata_with_shard_ranges_bis(self):
 
@@ -346,7 +419,7 @@ Shard Ranges (3):
         info['is_deleted'] = False
         out = StringIO()
         with mock.patch('sys.stdout', out):
-            print_db_info_metadata('container', info, {})
+            print_db_info_metadata('container', info, {}, verbose=True)
         if six.PY2:
             s_a = '\\xe3\\x82\\xa2'
             s_ya = '\\xe3\\x83\\xa4'
@@ -378,6 +451,10 @@ Sharding Metadata:
   Type: root
   State: sharded
 Shard Ranges (3):
+  States:
+        found: 1
+      created: 1
+      cleaved: 1
   Name: .sharded_a/shard_range_1
     lower: '1%s', upper: '1%s'
     Object Count: 1, Bytes Used: 1, State: cleaved (30)