28be6108c9
zuul sends the destination address without a trailing slash. This is because the LOG_PATH does not contain one. Check to see if we need to add a slash to object paths before pushing to swift Change-Id: I3e43c4ea266b57028fcaf9aa5fd09f1f5e37c306
164 lines
5.8 KiB
Python
Executable File
164 lines
5.8 KiB
Python
Executable File
#!/usr/bin/python
|
|
#
|
|
# Copyright 2014 Rackspace Australia
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""
|
|
Utility to upload folders to swift using the form post middleware
|
|
credentials provided by zuul
|
|
"""
|
|
|
|
import argparse
|
|
import magic
|
|
import os
|
|
import requests
|
|
import tempfile
|
|
|
|
|
|
def generate_log_index(file_list, logserver_prefix, swift_destination_prefix):
|
|
"""Create an index of logfiles and links to them"""
|
|
|
|
output = '<html><head><title>Index of results</title></head><body>'
|
|
output += '<ul>'
|
|
for f in file_list:
|
|
file_url = os.path.join(logserver_prefix, swift_destination_prefix,
|
|
f['filename'])
|
|
|
|
output += '<li>'
|
|
output += '<a href="%s">%s</a>' % (file_url, f['filename'])
|
|
output += '</li>'
|
|
|
|
output += '</ul>'
|
|
output += '</body></html>'
|
|
return output
|
|
|
|
|
|
def make_index_file(file_list, logserver_prefix, swift_destination_prefix,
|
|
index_filename='index.html'):
|
|
"""Writes an index into a file for pushing"""
|
|
|
|
index_content = generate_log_index(file_list, logserver_prefix,
|
|
swift_destination_prefix)
|
|
tempdir = tempfile.mkdtemp()
|
|
fd = open(os.path.join(tempdir, index_filename), 'w')
|
|
fd.write(index_content)
|
|
return os.path.join(tempdir, index_filename)
|
|
|
|
|
|
def get_file_mime(file_path):
|
|
"""Get the file mime using libmagic"""
|
|
|
|
if not os.path.isfile(file_path):
|
|
return None
|
|
|
|
if hasattr(magic, 'from_file'):
|
|
return magic.from_file(file_path, mime=True)
|
|
else:
|
|
# no magic.from_file, we might be using the libmagic bindings
|
|
m = magic.open(magic.MIME)
|
|
m.load()
|
|
return m.file(file_path).split(';')[0]
|
|
|
|
|
|
def swift_form_post_submit(file_list, url, hmac_body, signature):
|
|
"""Send the files to swift via the FormPost middleware"""
|
|
|
|
# We are uploading the file_list as an HTTP POST multipart encoded.
|
|
# First grab out the information we need to send back from the hmac_body
|
|
payload = {}
|
|
|
|
(object_prefix,
|
|
payload['redirect'],
|
|
payload['max_file_size'],
|
|
payload['max_file_count'],
|
|
payload['expires']) = hmac_body.split('\n')
|
|
payload['signature'] = signature
|
|
|
|
if len(file_list) > payload['max_file_count']:
|
|
# We can't upload this many files! We'll do what we can but the job
|
|
# should be reconfigured
|
|
file_list = file_list[:payload['max_file_count']]
|
|
|
|
files = {}
|
|
|
|
# Zuul's log path is generated without a tailing slash. As such the
|
|
# object prefix does not contain a slash and the files would be
|
|
# uploaded as 'prefix' + 'filename'. Assume we want the destination
|
|
# url to look like a folder and make sure there's a slash between.
|
|
filename_prefix = '/' if url[-1] != '/' else ''
|
|
for i, f in enumerate(file_list):
|
|
files['file%d' % (i + 1)] = (filename_prefix + f['filename'],
|
|
open(f['path'], 'rb'),
|
|
get_file_mime(f['path']))
|
|
|
|
requests.post(url, data=payload, files=files)
|
|
|
|
|
|
def zuul_swift_upload(file_path, swift_url, swift_hmac_body, swift_signature,
|
|
logserver_prefix, swift_destination_prefix):
|
|
"""Upload to swift using instructions from zuul"""
|
|
|
|
# file_list: a list of dicts with {path=..., filename=...} where filename
|
|
# is appended to the end of the object (paths can be used)
|
|
file_list = []
|
|
if os.path.isfile(file_path):
|
|
file_list.append({'filename': os.path.basename(file_path),
|
|
'path': file_path})
|
|
index_file = file_path
|
|
elif os.path.isdir(file_path):
|
|
for path, folders, files in os.walk(file_path):
|
|
for f in files:
|
|
full_path = os.path.join(path, f)
|
|
relative_name = os.path.relpath(full_path, file_path)
|
|
file_list.append({'filename': relative_name,
|
|
'path': full_path})
|
|
index_file = make_index_file(file_list, logserver_prefix,
|
|
swift_destination_prefix)
|
|
file_list.append({'filename': os.path.basename(index_file),
|
|
'path': index_file})
|
|
|
|
swift_form_post_submit(file_list, swift_url, swift_hmac_body,
|
|
swift_signature)
|
|
|
|
return os.path.join(logserver_prefix, swift_destination_prefix,
|
|
os.path.basename(index_file))
|
|
|
|
|
|
def grab_args():
|
|
"""Grab and return arguments"""
|
|
parser = argparse.ArgumentParser(
|
|
description="Upload results to swift using instructions from zuul"
|
|
)
|
|
parser.add_argument('-n', '--name', default="logs",
|
|
help='The instruction-set to use')
|
|
parser.add_argument('files', nargs='+', help='the file(s) to upload')
|
|
|
|
return parser.parse_args()
|
|
|
|
if __name__ == '__main__':
|
|
args = grab_args()
|
|
for file_path in args.files:
|
|
try:
|
|
result_url = zuul_swift_upload(
|
|
file_path,
|
|
os.environ['SWIFT_%s_URL' % args.name],
|
|
os.environ['SWIFT_%s_HMAC_BODY' % args.name],
|
|
os.environ['SWIFT_%s_SIGNATURE' % args.name],
|
|
os.environ['SWIFT_%s_LOGSERVER_PREFIX' % args.name],
|
|
os.environ['LOG_PATH']
|
|
)
|
|
print result_url
|
|
except KeyError as e:
|
|
print 'Environment variable %s not found' % e
|