Optionally display listings in raw json

Symlinks have recently added some new keys to container listings.  It's
very convenient to be able to see and reason about the extra information
in container listings.

Allowing raw json output is similar with what the client already does
for the info command, and it's forward compatible with any listing
enhancements added by future middleware development.

Change-Id: I88fb38529342ac4e4198aeccd2f10c69c7396704

swiftclient/shell.py

@@ -33,7 +33,8 @@ from sys import argv as sys_argv, exit, stderr, stdin
 from time import gmtime, strftime
 
 from swiftclient import RequestException
-from swiftclient.utils import config_true_value, generate_temp_url, prt_bytes
+from swiftclient.utils import config_true_value, generate_temp_url, \
+    prt_bytes, JSONableIterable
 from swiftclient.multithreading import OutputManager
 from swiftclient.exceptions import ClientException
 from swiftclient import __version__ as client_version
@@ -578,6 +579,8 @@ def st_list(parser, args, output_manager, return_parser=False):
         help='Roll up items with the given delimiter. For containers '
              'only. See OpenStack Swift API documentation for '
              'what this means.')
+    parser.add_argument('-j', '--json', action='store_true',
+                        help='print listing information in json')
     parser.add_argument(
         '-H', '--header', action='append', dest='header',
         default=[],
@@ -616,6 +619,20 @@ def st_list(parser, args, output_manager, return_parser=False):
                 else:
                     stats_parts_gen = swift.list(container=container)
 
+            if options.get('json', False):
+                def listing(stats_parts_gen=stats_parts_gen):
+                    for stats in stats_parts_gen:
+                        if stats["success"]:
+                            for item in stats['listing']:
+                                yield item
+                        else:
+                            raise stats["error"]
+
+                json.dump(
+                    JSONableIterable(listing()), output_manager.print_stream,
+                    sort_keys=True, indent=2)
+                output_manager.print_msg('')
+                return
             for stats in stats_parts_gen:
                 if stats["success"]:
                     _print_stats(options, stats, human)

swiftclient/utils.py

@@ -403,3 +403,25 @@ def normalize_manifest_path(path):
     if path.startswith('/'):
         return path[1:]
     return path
+
+
+class JSONableIterable(list):
+    def __init__(self, iterable):
+        self._iterable = iter(iterable)
+        try:
+            self._peeked = next(self._iterable)
+            self._has_items = True
+        except StopIteration:
+            self._peeked = None
+            self._has_items = False
+
+    def __bool__(self):
+        return self._has_items
+
+    __nonzero__ = __bool__
+
+    def __iter__(self):
+        if self._has_items:
+            yield self._peeked
+        for item in self._iterable:
+            yield item

tests/unit/test_shell.py

@@ -297,6 +297,26 @@ class TestShell(unittest.TestCase):
             mock.call('container', 'object',
                       headers={'Skip-Middleware': 'Test'})])
 
+    @mock.patch('swiftclient.service.Connection')
+    def test_list_json(self, connection):
+        connection.return_value.get_account.side_effect = [
+            [None, [{'name': 'container'}]],
+            [None, [{'name': u'\u263A', 'some-custom-key': 'and value'}]],
+            [None, []],
+        ]
+
+        argv = ["", "list", "--json"]
+        with CaptureOutput(suppress_systemexit=True) as output:
+            swiftclient.shell.main(argv)
+        calls = [mock.call(marker='', prefix=None, headers={}),
+                 mock.call(marker='container', prefix=None, headers={})]
+        connection.return_value.get_account.assert_has_calls(calls)
+
+        listing = [{'name': 'container'},
+                   {'name': u'\u263A', 'some-custom-key': 'and value'}]
+        expected = json.dumps(listing, sort_keys=True, indent=2) + '\n'
+        self.assertEqual(output.out, expected)
+
     @mock.patch('swiftclient.service.Connection')
     def test_list_account(self, connection):
         # Test account listing

tests/unit/test_utils.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import gzip
+import json
 import unittest
 import mock
 import six
@@ -638,3 +639,41 @@ class TestGetBody(unittest.TestCase):
             {'content-encoding': 'gzip'},
             buf.getvalue())
         self.assertEqual({'test': u'\u2603'}, result)
+
+
+class JSONTracker(object):
+    def __init__(self, data):
+        self.data = data
+        self.calls = []
+
+    def __iter__(self):
+        for item in self.data:
+            self.calls.append(('read', item))
+            yield item
+
+    def write(self, s):
+        self.calls.append(('write', s))
+
+
+class TestJSONableIterable(unittest.TestCase):
+    def test_json_dump_iterencodes(self):
+        t = JSONTracker([1, 'fish', 2, 'fish'])
+        json.dump(u.JSONableIterable(t), t)
+        self.assertEqual(t.calls, [
+            ('read', 1),
+            ('write', '[1'),
+            ('read', 'fish'),
+            ('write', ', "fish"'),
+            ('read', 2),
+            ('write', ', 2'),
+            ('read', 'fish'),
+            ('write', ', "fish"'),
+            ('write', ']'),
+        ])
+
+    def test_json_dump_empty_iter(self):
+        t = JSONTracker([])
+        json.dump(u.JSONableIterable(t), t)
+        self.assertEqual(t.calls, [
+            ('write', '[]'),
+        ])