Print file sizes in indexes for on disk files
File sizes are an important piece of information in log file indexes as they help readers pick out inappropriately sized files which may indicate the bug can be found there. Add file sizes to the index so that we don't lose this info when compared to mod_autoindex. Change-Id: I248a4fc9bd5ff3fa536e30bcc981a2d5f640307c
This commit is contained in:
parent
a6fa3bfd2d
commit
a9b91f6a05
|
@ -91,6 +91,16 @@ def safe_path(root, log_name):
|
|||
return None
|
||||
|
||||
|
||||
def sizeof_fmt(num, suffix='B'):
|
||||
# From http://stackoverflow.com/questions/1094841/
|
||||
# reusable-library-to-get-human-readable-version-of-file-size
|
||||
for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
|
||||
if abs(num) < 1024.0:
|
||||
return "%3.1f%s%s" % (num, unit, suffix)
|
||||
num /= 1024.0
|
||||
return "%.1f%s%s" % (num, 'Y', suffix)
|
||||
|
||||
|
||||
def _get_swift_connection(swift_config):
|
||||
# TODO(jhesketh): refactor the generator into a class so we can keep a
|
||||
# persistent connection. For now, emulate a static variable on this method
|
||||
|
@ -213,18 +223,22 @@ class IndexIterableBuffer(collections.Iterable):
|
|||
# Use sets here to dedup. We can have duplicates
|
||||
# if disk and swift based paths have overlap.
|
||||
file_set = self.disk_list() | self.swift_list()
|
||||
# file_list is a list of tuples (relpath, name)
|
||||
# file_list is a list of tuples (relpath, name, size)
|
||||
self.file_list = sorted(file_set, key=lambda tup: tup[0])
|
||||
|
||||
def disk_list(self):
|
||||
file_set = set()
|
||||
if os.path.isdir(self.logpath):
|
||||
for f in os.listdir(self.logpath):
|
||||
if os.path.isdir(os.path.join(self.logpath, f)):
|
||||
full_path = os.path.join(self.logpath, f)
|
||||
stat_info = os.stat(full_path)
|
||||
size = sizeof_fmt(stat_info.st_size)
|
||||
if os.path.isdir(full_path):
|
||||
f = f + '/' if f[-1] != '/' else f
|
||||
file_set.add((
|
||||
os.path.join('/', self.logname, f),
|
||||
f
|
||||
f,
|
||||
size
|
||||
))
|
||||
return file_set
|
||||
|
||||
|
@ -248,9 +262,11 @@ class IndexIterableBuffer(collections.Iterable):
|
|||
fname
|
||||
else:
|
||||
fname = os.path.relpath(f['name'], self.logname)
|
||||
# TODO(clarkb) get size from swift and write it here.
|
||||
file_set.add((
|
||||
os.path.join('/', self.logname, fname),
|
||||
fname
|
||||
fname,
|
||||
"NA"
|
||||
))
|
||||
except Exception:
|
||||
import traceback
|
||||
|
|
|
@ -5,10 +5,10 @@
|
|||
</head>
|
||||
<body>
|
||||
<h1>Index of {{ logname }}</h1>
|
||||
<ul>
|
||||
{% for link, title in file_list %}
|
||||
<li><a href="{{ link }}">{{ title }}</a></li>
|
||||
<table><tr><th>Name</th><th>Size</th></tr>
|
||||
{% for link, title, size in file_list %}
|
||||
<tr><td><a href="{{ link }}">{{ title }}</a></td><td style="text-align: right">{{ size }}</td></tr>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
@ -98,7 +98,7 @@ def fake_get_object(self, container, name, resp_chunk_size=None):
|
|||
return resp_headers, object_body
|
||||
|
||||
|
||||
def fake_get_container_factory(_swift_index_items=[]):
|
||||
def fake_get_container_factory(_swift_index_items=None):
|
||||
def fake_get_container(self, container, prefix=None, delimiter=None):
|
||||
index_items = []
|
||||
if _swift_index_items:
|
||||
|
@ -107,7 +107,7 @@ def fake_get_container_factory(_swift_index_items=[]):
|
|||
index_items.append({'subdir': os.path.join(prefix, i)})
|
||||
else:
|
||||
index_items.append({'name': os.path.join(prefix, i)})
|
||||
else:
|
||||
elif _swift_index_items == []:
|
||||
name = prefix[len('non-existent/'):]
|
||||
p = os.path.join(base.samples_path('samples'), name)
|
||||
for i in os.listdir(p):
|
||||
|
@ -116,7 +116,9 @@ def fake_get_container_factory(_swift_index_items=[]):
|
|||
{'subdir': os.path.join(prefix, i + '/')})
|
||||
else:
|
||||
index_items.append({'name': os.path.join(prefix, i)})
|
||||
|
||||
else:
|
||||
# No swift container data.
|
||||
pass
|
||||
return {}, index_items
|
||||
return fake_get_container
|
||||
|
||||
|
@ -300,12 +302,14 @@ class TestWsgiDisk(base.TestCase):
|
|||
self.assertEqual('<!DOCTYPE html>', full_lines[0])
|
||||
self.assertIn('samples/</title>', full_lines[3])
|
||||
self.assertEqual(
|
||||
' <li><a href="/samples/console.html.gz">'
|
||||
'console.html.gz</a></li>',
|
||||
' <tr><td><a href="/samples/console.html.gz">'
|
||||
'console.html.gz</a></td><td style="text-align: right">'
|
||||
'277.4KB</td></tr>',
|
||||
full_lines[9])
|
||||
self.assertEqual(
|
||||
' <li><a href="/samples/wsgi_plain.conf">'
|
||||
'wsgi_plain.conf</a></li>',
|
||||
' <tr><td><a href="/samples/wsgi_plain.conf">'
|
||||
'wsgi_plain.conf</a></td><td style="text-align: right">'
|
||||
'177.0B</td></tr>',
|
||||
full_lines[-5])
|
||||
self.assertEqual('</html>', full_lines[-1])
|
||||
|
||||
|
@ -397,7 +401,7 @@ class TestWsgiSwift(TestWsgiDisk):
|
|||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_container',
|
||||
fake_get_container_factory())
|
||||
fake_get_container_factory([]))
|
||||
def test_folder_index(self):
|
||||
self.wsgi_config_file = (base.samples_path('samples') +
|
||||
'wsgi_folder_index.conf')
|
||||
|
@ -410,12 +414,14 @@ class TestWsgiSwift(TestWsgiDisk):
|
|||
self.assertEqual('<!DOCTYPE html>', full_lines[0])
|
||||
self.assertIn('non-existent/</title>', full_lines[3])
|
||||
self.assertEqual(
|
||||
' <li><a href="/non-existent/console.html.gz">'
|
||||
'console.html.gz</a></li>',
|
||||
' <tr><td><a href="/non-existent/console.html.gz">'
|
||||
'console.html.gz</a></td><td style="text-align: right">'
|
||||
'NA</td></tr>',
|
||||
full_lines[9])
|
||||
self.assertEqual(
|
||||
' <li><a href="/non-existent/wsgi_plain.conf">'
|
||||
'wsgi_plain.conf</a></li>',
|
||||
' <tr><td><a href="/non-existent/wsgi_plain.conf">'
|
||||
'wsgi_plain.conf</a></td><td style="text-align: right">'
|
||||
'NA</td></tr>',
|
||||
full_lines[-5])
|
||||
self.assertEqual('</html>', full_lines[-1])
|
||||
|
||||
|
@ -436,21 +442,30 @@ class TestWsgiSwift(TestWsgiDisk):
|
|||
full_lines = full.split('\n')
|
||||
self.assertEqual('<!DOCTYPE html>', full_lines[0])
|
||||
self.assertIn('samples/</title>', full_lines[3])
|
||||
self.assertEqual(' <li><a href="/samples/a">a</a></li>',
|
||||
full_lines[9])
|
||||
self.assertEqual(' <li><a href="/samples/b">b</a></li>',
|
||||
full_lines[11])
|
||||
self.assertEqual(
|
||||
' <li><a href="/samples/console.html.gz">'
|
||||
'console.html.gz</a></li>',
|
||||
' <tr><td><a href="/samples/a">a</a></td>'
|
||||
'<td style="text-align: right">NA</td></tr>',
|
||||
full_lines[9])
|
||||
self.assertEqual(
|
||||
' <tr><td><a href="/samples/b">b</a></td>'
|
||||
'<td style="text-align: right">NA</td></tr>',
|
||||
full_lines[11])
|
||||
self.assertEqual(
|
||||
' <tr><td><a href="/samples/console.html.gz">'
|
||||
'console.html.gz</a></td><td style="text-align: right">'
|
||||
'277.4KB</td></tr>',
|
||||
full_lines[13])
|
||||
self.assertEqual(
|
||||
' <li><a href="/samples/dir/">dir/</a></li>',
|
||||
' <tr><td><a href="/samples/dir/">dir/</a></td>'
|
||||
'<td style="text-align: right">NA</td></tr>',
|
||||
full_lines[17])
|
||||
self.assertEqual(
|
||||
' <li><a href="/samples/wsgi_plain.conf">'
|
||||
'wsgi_plain.conf</a></li>',
|
||||
' <tr><td><a href="/samples/wsgi_plain.conf">'
|
||||
'wsgi_plain.conf</a></td><td style="text-align: right">'
|
||||
'177.0B</td></tr>',
|
||||
full_lines[-7])
|
||||
self.assertEqual(' <li><a href="/samples/z">z</a></li>',
|
||||
full_lines[-5])
|
||||
self.assertEqual(
|
||||
' <tr><td><a href="/samples/z">z</a></td>'
|
||||
'<td style="text-align: right">NA</td></tr>',
|
||||
full_lines[-5])
|
||||
self.assertEqual('</html>', full_lines[-1])
|
||||
|
|
Loading…
Reference in New Issue