openstack/project-config
Code Issues Proposed changes

Merge "Add in parent links to zuul log indexes"

Browse Source
This commit is contained in:
Jenkins authored and Gerrit Code Review committed 2015-01-27 02:11:19 +00:00
2 parents d7446cec39 951c6bc439
commit 68f086e5ac
1 file changed
+150 -53
Download Patch File Download Diff File Expand all files Collapse all files
jenkins/scripts/zuul_swift_upload.py
+150 -53
View File
@@ -28,35 +28,42 @@ import tempfile
import time
def generate_log_index(file_list, logserver_prefix, swift_destination_prefix):
# file_detail format: A dictionary containing details of the file such as
# full url links or relative path names etc.
# Used to generate indexes with links or as the file path
# to push to swift.
# file_details = {
# 'filename': The base filename to appear in indexes,
# 'path': The path on the filesystem to the source file
# (absolute or relative),
# 'relative_name': The file relative name. Used as the object name
# in swift, is typically the rel path to the file
# list supplied,
# 'url': The URL to the log on the log server (absolute, supplied
# by logserver_prefix and swift_destination_prefix)
# }
def generate_log_index(folder_links):
"""Create an index of logfiles and links to them"""
output = '<html><head><title>Index of results</title></head><body>'
output += '<ul>'
for f in file_list:
file_url = os.path.join(logserver_prefix, swift_destination_prefix, f)
# Because file_list is simply a list to create an index for and it
# isn't necessarily on disk we can't check if a file is a folder or
# not. As such we normalise the name to get the folder/filename but
# then need to check if the last character was a trailing slash so to
# re-append it to make it obvious that it links to a folder
filename_postfix = '/' if f[-1] == '/' else ''
filename = os.path.basename(os.path.normpath(f)) + filename_postfix
output = '<html><head><title>Index of results</title></head><body>\n'
output += '<ul>\n'
for file_details in folder_links:
output += '<li>'
output += '<a href="%s">%s</a>' % (file_url, filename)
output += '</li>'
output += '<a href="%s">%s</a>' % (file_details['url'],
file_details['filename'])
output += '</li>\n'
output += '</ul>'
output += '</ul>\n'
output += '</body></html>'
return output
def make_index_file(file_list, logserver_prefix, swift_destination_prefix,
index_filename='index.html'):
def make_index_file(folder_links, index_filename='index.html'):
"""Writes an index into a file for pushing"""
index_content = generate_log_index(file_list, logserver_prefix,
swift_destination_prefix)
index_content = generate_log_index(folder_links)
tempdir = tempfile.mkdtemp()
fd = open(os.path.join(tempdir, index_filename), 'w')
fd.write(index_content)
@@ -99,9 +106,9 @@ def swift_form_post_submit(file_list, url, hmac_body, signature):
if payload['expires'] < time.time():
raise Exception("Ran out of time uploading files!")
files = {}
# Zuul's log path is generated without a tailing slash. As such the
# object prefix does not contain a slash and the files would be
# uploaded as 'prefix' + 'filename'. Assume we want the destination
# Zuul's log path is sometimes generated without a tailing slash. As
# such the object prefix does not contain a slash and the files would
# be uploaded as 'prefix' + 'filename'. Assume we want the destination
# url to look like a folder and make sure there's a slash between.
filename_prefix = '/' if url[-1] != '/' else ''
for i, f in enumerate(sub_file_list):
@@ -109,50 +116,114 @@ def swift_form_post_submit(file_list, url, hmac_body, signature):
sys.stderr.write('Warning: %s exceeds %d bytes. Skipping...\n'
% (f['path'], int(payload['max_file_size'])))
continue
files['file%d' % (i + 1)] = (filename_prefix + f['filename'],
files['file%d' % (i + 1)] = (filename_prefix + f['relative_name'],
open(f['path'], 'rb'),
get_file_mime(f['path']))
requests.post(url, data=payload, files=files)
def build_file_list(file_path, logserver_prefix, swift_destination_prefix,
create_dir_indexes=True):
create_dir_indexes=True, create_parent_links=True):
"""Generate a list of files to upload to zuul. Recurses through directories
and generates index.html files if requested."""
# file_list: a list of dicts with {path=..., filename=...} where filename
# is appended to the end of the object (paths can be used)
# file_list: A list of files to push to swift (in file_detail format)
file_list = []
destination_prefix = os.path.join(logserver_prefix,
swift_destination_prefix)
if os.path.isfile(file_path):
file_list.append({'filename': os.path.basename(file_path),
'path': file_path})
filename = os.path.basename(file_path)
full_path = file_path
relative_name = filename
url = os.path.join(destination_prefix, filename)
file_details = {
'filename': filename,
'path': full_path,
'relative_name': relative_name,
'url': url,
}
file_list.append(file_details)
elif os.path.isdir(file_path):
if file_path[-1] == os.sep:
file_path = file_path[:-1]
parent_dir = os.path.dirname(file_path)
for path, folders, files in os.walk(file_path):
folder_contents = []
for f in files:
full_path = os.path.join(path, f)
# relative_path: The path between the given director and the one
# being currently walked.
relative_path = os.path.relpath(path, parent_dir)
# folder_links: A list of files and their links to generate an
# index from if required (in file_detail format)
folder_links = []
# Place a link to the parent directory?
if create_parent_links:
filename = '../'
full_path = os.path.normpath(os.path.join(path, filename))
relative_name = os.path.relpath(full_path, parent_dir)
push_file = {'filename': relative_name,
'path': full_path}
file_list.append(push_file)
folder_contents.append(relative_name)
if relative_name == '.':
relative_name = ''
url = os.path.join(destination_prefix, relative_name)
file_details = {
'filename': filename,
'path': full_path,
'relative_name': relative_name,
'url': url,
}
folder_links.append(file_details)
for f in files:
filename = os.path.basename(f)
full_path = os.path.join(path, filename)
relative_name = os.path.relpath(full_path, parent_dir)
url = os.path.join(destination_prefix, relative_name)
file_details = {
'filename': filename,
'path': full_path,
'relative_name': relative_name,
'url': url,
}
file_list.append(file_details)
folder_links.append(file_details)
for f in folders:
full_path = os.path.join(path, f)
filename = os.path.basename(f) + '/'
full_path = os.path.join(path, filename)
relative_name = os.path.relpath(full_path, parent_dir)
folder_contents.append(relative_name + '/')
url = os.path.join(destination_prefix, relative_name)
file_details = {
'filename': filename,
'path': full_path,
'relative_name': relative_name,
'url': url,
}
folder_links.append(file_details)
if create_dir_indexes:
index_file = make_index_file(folder_contents, logserver_prefix,
swift_destination_prefix)
relative_name = os.path.relpath(path, parent_dir)
file_list.append({
'filename': os.path.join(relative_name,
os.path.basename(index_file)),
'path': index_file})
full_path = make_index_file(folder_links)
filename = os.path.basename(full_path)
relative_name = os.path.join(relative_path, filename)
url = os.path.join(destination_prefix, relative_name)
file_details = {
'filename': filename,
'path': full_path,
'relative_name': relative_name,
'url': url,
}
file_list.append(file_details)
return file_list
@@ -168,6 +239,8 @@ def grab_args():
help='do not generate a root index')
parser.add_argument('--no-dir-indexes', action='store_true',
help='do not generate a indexes inside dirs')
parser.add_argument('--no-parent-links', action='store_true',
help='do not include links back to a parent dir')
parser.add_argument('-n', '--name', default="logs",
help='The instruction-set to use')
parser.add_argument('files', nargs='+', help='the file(s) to upload')
@@ -177,8 +250,11 @@ def grab_args():
if __name__ == '__main__':
args = grab_args()
# file_list: A list of files to push to swift (in file_detail format)
file_list = []
root_list = []
# folder_links: A list of files and their links to generate an
# index from if required (in file_detail format)
folder_links = []
try:
logserver_prefix = os.environ['SWIFT_%s_LOGSERVER_PREFIX' % args.name]
@@ -190,25 +266,46 @@ if __name__ == '__main__':
print 'Environment variable %s not found' % e
quit()
destination_prefix = os.path.join(logserver_prefix,
swift_destination_prefix)
for file_path in args.files:
file_path = os.path.normpath(file_path)
if os.path.isfile(file_path):
root_list.append(os.path.basename(file_path))
filename = os.path.basename(file_path)
else:
root_list.append(os.path.basename(file_path) + '/')
filename = os.path.basename(file_path) + '/'
url = os.path.join(destination_prefix, filename)
file_details = {
'filename': filename,
'path': file_path,
'relative_name': filename,
'url': url,
}
folder_links.append(file_details)
file_list += build_file_list(
file_path, logserver_prefix, swift_destination_prefix,
(not (args.no_indexes or args.no_dir_indexes))
(not (args.no_indexes or args.no_dir_indexes)),
(not args.no_parent_links)
)
index_file = ''
if not (args.no_indexes or args.no_root_index):
index_file = make_index_file(root_list, logserver_prefix,
swift_destination_prefix)
file_list.append({
'filename': os.path.basename(index_file),
'path': index_file})
full_path = make_index_file(folder_links)
filename = os.path.basename(full_path)
relative_name = filename
url = os.path.join(destination_prefix, relative_name)
file_details = {
'filename': filename,
'path': full_path,
'relative_name': relative_name,
'url': url,
}
file_list.append(file_details)
swift_form_post_submit(file_list, swift_url, swift_hmac_body,
swift_signature)