Test role for upload-logs-swift
This creates a testing role for upload-logs-swift so that we can test chagnes to this role before moving them into the production role. Change-Id: If55b0e2809d9309bc98e6763da32219d996dbfa4
This commit is contained in:
Clark Boylan
parent
3c60b35a19
commit
61c05a180f
|
|
@ -3,3 +3,4 @@ Deprecrated and Test Roles
|
|||
|
||||
.. zuul:autorole:: fetch-zuul-cloner
|
||||
.. zuul:autorole:: test-mirror-workspace-git-repos
|
||||
.. zuul:autorole:: test-upload-logs-swift
|
||||
|
|
|
|||
|
|
@ -0,0 +1,80 @@
|
|||
THIS IS FOR TESTING ONLY
|
||||
|
||||
Upload logs to a swift container
|
||||
|
||||
This uploads logs to an OpenStack Object Store (Swift) container.
|
||||
|
||||
**Role Variables**
|
||||
|
||||
.. zuul:rolevar:: zuul_site_upload_logs
|
||||
:default: true
|
||||
|
||||
Controls when logs are uploaded. true, the default, means always
|
||||
upload logs. false means never upload logs. 'failure' means to only
|
||||
upload logs when the job has failed.
|
||||
|
||||
.. note:: Intended to be set by admins via site-variables.
|
||||
|
||||
.. zuul:rolevar:: zuul_log_cloud_config
|
||||
|
||||
Complex argument which contains the cloud configuration in
|
||||
os-cloud-config (clouds.yaml) format. It is expected that this
|
||||
argument comes from a `Secret`.
|
||||
|
||||
.. zuul:rolevar:: zuul_log_partition
|
||||
:default: false
|
||||
|
||||
If set to true, then the first component of the log path will be
|
||||
removed from the object name and added to the container name, so
|
||||
that logs for different changes are distributed across a large
|
||||
number of containers.
|
||||
|
||||
.. zuul:rolevar:: zuul_log_container
|
||||
:default: logs
|
||||
|
||||
This role will create containers which do not already exist. If
|
||||
partitioning is not enabled, this is the name of the container
|
||||
which will be used. If partitioning is enabled, then this will be
|
||||
used as the prefix for the container name which will be separated
|
||||
from the partition name by an underscore. For example, "logs_42"
|
||||
would be the container name for partition 42.
|
||||
|
||||
Note that you will want to set this to a value that uniquely
|
||||
identifies your Zuul installation if using shared object stores that
|
||||
require globally unique container names. For example if using a
|
||||
public cloud whose Swift API is provided by Ceph.
|
||||
|
||||
.. zuul:rolevar:: zuul_log_container_public
|
||||
:default: true
|
||||
|
||||
If the container is created, should it be created with global read
|
||||
ACLs. If the container already exists, it will not be modified.
|
||||
|
||||
.. zuul:rolevar:: zuul_log_delete_after
|
||||
:default: 15552000
|
||||
|
||||
Number of seconds to delete objects after upload. Default is 6 months
|
||||
(15552000 seconds) and if set to 0 X-Delete-After will not be set.
|
||||
|
||||
.. zuul:rolevar:: zuul_log_path
|
||||
:default: Generated by the role `set-zuul-log-path-fact`
|
||||
|
||||
Prepend this path to the object names when uploading.
|
||||
|
||||
.. zuul:rolevar:: zuul_log_create_indexes
|
||||
:default: true
|
||||
|
||||
Whether to create `index.html` files with directory indexes. If set
|
||||
to false, Swift containers can be marked with a `Web-Listings=true`
|
||||
property to activate Swift's own directory indexing.
|
||||
|
||||
.. zuul:rolevar:: zuul_log_path_shard_build
|
||||
:default: False
|
||||
|
||||
This var is consumed by set-zuul-log-path-fact which upload-logs-swift
|
||||
calls into. If you set this you will get log paths prefixed with the
|
||||
first three characters of the build uuid. This will improve log file
|
||||
sharding.
|
||||
|
||||
More details can be found at
|
||||
:zuul:rolevar:`set-zuul-log-path-fact.zuul_log_path_shard_build`.
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
zuul_log_partition: false
|
||||
zuul_log_container: logs
|
||||
zuul_log_container_public: true
|
||||
zuul_log_create_indexes: true
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
#!/usr/bin/env python3
|
||||
#
|
||||
# Copyright 2019 Red Hat, Inc
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
import openstack
|
||||
import requests
|
||||
import logging
|
||||
import os
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
# logging.getLogger("requests").setLevel(logging.DEBUG)
|
||||
# logging.getLogger("keystoneauth").setLevel(logging.INFO)
|
||||
# logging.getLogger("stevedore").setLevel(logging.INFO)
|
||||
logging.captureWarnings(True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Delete a swift container"
|
||||
)
|
||||
parser.add_argument('cloud',
|
||||
help='Name of the cloud to use when uploading')
|
||||
parser.add_argument('container',
|
||||
help='Name of the container to use when uploading')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
cloud = openstack.connect(cloud=args.cloud)
|
||||
|
||||
sess = cloud.config.get_session()
|
||||
adapter = requests.adapters.HTTPAdapter(pool_maxsize=100)
|
||||
sess.mount('https://', adapter)
|
||||
|
||||
container = cloud.get_container(args.container)
|
||||
print('Found container', container)
|
||||
print()
|
||||
for x in cloud.object_store.objects(args.container):
|
||||
print('Delete object', x.name)
|
||||
if x.name == '/':
|
||||
endpoint = cloud.object_store.get_endpoint()
|
||||
container = os.path.join(endpoint, args.container)
|
||||
cloud.session.delete(container + '//')
|
||||
else:
|
||||
cloud.object_store.delete_object(x)
|
||||
|
||||
print()
|
||||
print('Delete container', container)
|
||||
cloud.object_store.delete_container(args.container)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1 @@
|
|||
{"test": "foo"}
|
||||
Binary file not shown.
|
|
@ -0,0 +1,3 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 52 B |
Binary file not shown.
|
|
@ -0,0 +1,2 @@
|
|||
This is a plan text file with a funny name.
|
||||
The index links should escape the :'s.
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"test": "foo"}
|
||||
|
|
@ -0,0 +1,404 @@
|
|||
# Copyright (C) 2018 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Make coding more python3-ish
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import testtools
|
||||
import time
|
||||
import stat
|
||||
import fixtures
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from .zuul_swift_upload import FileList, Indexer, FileDetail
|
||||
|
||||
|
||||
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
|
||||
'test-fixtures')
|
||||
|
||||
|
||||
class SymlinkFixture(fixtures.Fixture):
|
||||
links = [
|
||||
('bad_symlink', '/etc'),
|
||||
('bad_symlink_file', '/etc/issue'),
|
||||
('good_symlink', 'controller'),
|
||||
('recursive_symlink', '.'),
|
||||
('symlink_file', 'job-output.json'),
|
||||
('symlink_loop_a', 'symlink_loop'),
|
||||
('symlink_loop/symlink_loop_b', '..'),
|
||||
]
|
||||
|
||||
def _setUp(self):
|
||||
self._cleanup()
|
||||
for (src, target) in self.links:
|
||||
path = os.path.join(FIXTURE_DIR, 'links', src)
|
||||
os.symlink(target, path)
|
||||
self.addCleanup(self._cleanup)
|
||||
|
||||
def _cleanup(self):
|
||||
for (src, target) in self.links:
|
||||
path = os.path.join(FIXTURE_DIR, 'links', src)
|
||||
if os.path.exists(path):
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
class TestFileList(testtools.TestCase):
|
||||
|
||||
def assert_files(self, result, files):
|
||||
self.assertEqual(len(result), len(files))
|
||||
for expected, received in zip(files, result):
|
||||
self.assertEqual(expected[0], received.relative_path)
|
||||
if expected[0] and expected[0][-1] == '/':
|
||||
efilename = os.path.split(
|
||||
os.path.dirname(expected[0]))[1] + '/'
|
||||
else:
|
||||
efilename = os.path.split(expected[0])[1]
|
||||
self.assertEqual(efilename, received.filename)
|
||||
if received.folder:
|
||||
if received.full_path is not None and expected[0] != '':
|
||||
self.assertTrue(os.path.isdir(received.full_path))
|
||||
else:
|
||||
self.assertTrue(os.path.isfile(received.full_path))
|
||||
self.assertEqual(expected[1], received.mimetype)
|
||||
self.assertEqual(expected[2], received.encoding)
|
||||
|
||||
def find_file(self, file_list, path):
|
||||
for f in file_list:
|
||||
if f.relative_path == path:
|
||||
return f
|
||||
|
||||
def test_single_dir_trailing_slash(self):
|
||||
'''Test a single directory with a trailing slash'''
|
||||
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
])
|
||||
|
||||
def test_single_dir(self):
|
||||
'''Test a single directory without a trailing slash'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('logs', 'application/directory', None),
|
||||
('logs/controller', 'application/directory', None),
|
||||
('logs/zuul-info', 'application/directory', None),
|
||||
('logs/job-output.json', 'application/json', None),
|
||||
('logs/controller/subdir', 'application/directory', None),
|
||||
('logs/controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('logs/controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('logs/controller/journal.xz', 'text/plain', 'xz'),
|
||||
('logs/controller/service_log.txt', 'text/plain', None),
|
||||
('logs/controller/syslog', 'text/plain', None),
|
||||
('logs/controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('logs/controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('logs/zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('logs/zuul-info/zuul-info.controller.txt',
|
||||
'text/plain', None),
|
||||
])
|
||||
|
||||
def test_single_file(self):
|
||||
'''Test a single file'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR,
|
||||
'logs/zuul-info/inventory.yaml'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('inventory.yaml', 'text/plain', None),
|
||||
])
|
||||
|
||||
def test_symlinks(self):
|
||||
'''Test symlinks'''
|
||||
with FileList() as fl:
|
||||
self.useFixture(SymlinkFixture())
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'links/'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('good_symlink', 'application/directory', None),
|
||||
('recursive_symlink', 'application/directory', None),
|
||||
('symlink_loop', 'application/directory', None),
|
||||
('symlink_loop_a', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
('symlink_file', 'text/plain', None),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('symlink_loop/symlink_loop_b', 'application/directory', None),
|
||||
('symlink_loop/placeholder', 'text/plain', None),
|
||||
])
|
||||
|
||||
def test_index_files(self):
|
||||
'''Test index generation'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes()
|
||||
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('index.html', 'text/html', None),
|
||||
('logs', 'application/directory', None),
|
||||
('logs/controller', 'application/directory', None),
|
||||
('logs/zuul-info', 'application/directory', None),
|
||||
('logs/job-output.json', 'application/json', None),
|
||||
('logs/index.html', 'text/html', None),
|
||||
('logs/controller/subdir', 'application/directory', None),
|
||||
('logs/controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('logs/controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('logs/controller/journal.xz', 'text/plain', 'xz'),
|
||||
('logs/controller/service_log.txt', 'text/plain', None),
|
||||
('logs/controller/syslog', 'text/plain', None),
|
||||
('logs/controller/index.html', 'text/html', None),
|
||||
('logs/controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('logs/controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('logs/controller/subdir/index.html', 'text/html', None),
|
||||
('logs/zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('logs/zuul-info/zuul-info.controller.txt',
|
||||
'text/plain', None),
|
||||
('logs/zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
|
||||
self.assertEqual(len(rows), 1)
|
||||
|
||||
self.assertEqual(rows[0].find('a').get('href'), 'logs/')
|
||||
self.assertEqual(rows[0].find('a').text, 'logs/')
|
||||
|
||||
subdir_index = self.find_file(
|
||||
fl, 'logs/controller/subdir/index.html')
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
self.assertEqual(rows[0].find('a').get('href'), '../')
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt')
|
||||
self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[2].find('a').text, 'subdir.txt')
|
||||
|
||||
def test_index_files_trailing_slash(self):
|
||||
'''Test index generation with a trailing slash'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes()
|
||||
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
|
||||
self.assertEqual(len(rows), 3)
|
||||
|
||||
self.assertEqual(rows[0].find('a').get('href'), 'controller/')
|
||||
self.assertEqual(rows[0].find('a').text, 'controller/')
|
||||
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
|
||||
self.assertEqual(rows[1].find('a').text, 'zuul-info/')
|
||||
|
||||
subdir_index = self.find_file(fl, 'controller/subdir/index.html')
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
self.assertEqual(rows[0].find('a').get('href'), '../')
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt')
|
||||
self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[2].find('a').text, 'subdir.txt')
|
||||
|
||||
def test_topdir_parent_link(self):
|
||||
'''Test index generation creates topdir parent link'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes(
|
||||
create_parent_links=True,
|
||||
create_topdir_parent_link=True)
|
||||
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
|
||||
self.assertEqual(len(rows), 4)
|
||||
|
||||
self.assertEqual(rows[0].find('a').get('href'), '../')
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'controller/')
|
||||
self.assertEqual(rows[1].find('a').text, 'controller/')
|
||||
|
||||
self.assertEqual(rows[2].find('a').get('href'), 'zuul-info/')
|
||||
self.assertEqual(rows[2].find('a').text, 'zuul-info/')
|
||||
|
||||
subdir_index = self.find_file(fl, 'controller/subdir/index.html')
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
self.assertEqual(rows[0].find('a').get('href'), '../')
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt')
|
||||
self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[2].find('a').text, 'subdir.txt')
|
||||
|
||||
def test_no_parent_links(self):
|
||||
'''Test index generation creates topdir parent link'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes(
|
||||
create_parent_links=False,
|
||||
create_topdir_parent_link=False)
|
||||
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
|
||||
self.assertEqual(len(rows), 3)
|
||||
|
||||
self.assertEqual(rows[0].find('a').get('href'), 'controller/')
|
||||
self.assertEqual(rows[0].find('a').text, 'controller/')
|
||||
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
|
||||
self.assertEqual(rows[1].find('a').text, 'zuul-info/')
|
||||
|
||||
subdir_index = self.find_file(fl, 'controller/subdir/index.html')
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(rows[0].find('a').get('href'), 'foo%3A%3A3.txt')
|
||||
self.assertEqual(rows[0].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[1].find('a').text, 'subdir.txt')
|
||||
|
||||
|
||||
class TestFileDetail(testtools.TestCase):
|
||||
|
||||
def test_get_file_detail(self):
|
||||
'''Test files info'''
|
||||
path = os.path.join(FIXTURE_DIR, 'logs/job-output.json')
|
||||
file_detail = FileDetail(path, '')
|
||||
path_stat = os.stat(path)
|
||||
self.assertEqual(
|
||||
time.gmtime(path_stat[stat.ST_MTIME]),
|
||||
file_detail.last_modified)
|
||||
self.assertEqual(16, file_detail.size)
|
||||
|
||||
def test_get_file_detail_missing_file(self):
|
||||
'''Test files that go missing during a walk'''
|
||||
|
||||
file_detail = FileDetail('missing/file/that/we/cant/find', '')
|
||||
|
||||
self.assertEqual(time.gmtime(0), file_detail.last_modified)
|
||||
self.assertEqual(0, file_detail.size)
|
||||
|
|
@ -0,0 +1,915 @@
|
|||
#!/usr/bin/env python3
|
||||
#
|
||||
# Copyright 2014 Rackspace Australia
|
||||
# Copyright 2018 Red Hat, Inc
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Make coding more python3-ish
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
|
||||
"""
|
||||
Utility to upload files to swift
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import io
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
try:
|
||||
import queue as queuelib
|
||||
except ImportError:
|
||||
import Queue as queuelib
|
||||
import shutil
|
||||
import stat
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
try:
|
||||
import urllib.parse as urlparse
|
||||
except ImportError:
|
||||
import urllib as urlparse
|
||||
import zlib
|
||||
import collections
|
||||
|
||||
import openstack
|
||||
import requests
|
||||
import requests.exceptions
|
||||
import requestsexceptions
|
||||
import keystoneauth1.exceptions
|
||||
|
||||
from ansible.module_utils.basic import AnsibleModule
|
||||
|
||||
try:
|
||||
# Python 3.3+
|
||||
from collections.abc import Sequence
|
||||
except ImportError:
|
||||
from collections import Sequence
|
||||
|
||||
mimetypes.init()
|
||||
mimetypes.add_type('text/plain', '.yaml')
|
||||
|
||||
MAX_UPLOAD_THREADS = 24
|
||||
POST_ATTEMPTS = 3
|
||||
|
||||
# Map mime types to apache icons
|
||||
APACHE_MIME_ICON_MAP = {
|
||||
'_default': 'unknown.png',
|
||||
'application/gzip': 'compressed.png',
|
||||
'application/directory': 'folder.png',
|
||||
'text/html': 'text.png',
|
||||
'text/plain': 'text.png',
|
||||
}
|
||||
|
||||
# Map mime types to apache icons
|
||||
APACHE_FILE_ICON_MAP = {
|
||||
'..': 'back.png',
|
||||
}
|
||||
|
||||
# These icon files are from the Apache project and are in the public
|
||||
# domain.
|
||||
ICON_IMAGES = {
|
||||
'back.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAElBMVEX/'
|
||||
'///M//+ZmZlmZmYzMzMAAACei5rnAAAAAnRSTlP/AOW3MEoAAABWSURB'
|
||||
'VHjabdBBCgAhDEPRRpv7X3kwEMsQ//IRRC08urjRHbha5VLFUsVSxVI9'
|
||||
'lmDh5hMpHD6n0EgoiZG0DNINpnWlcVXaRix76e1/8dddcL6nG0Ri9gHj'
|
||||
'tgSXKYeLBgAAAABJRU5ErkJggg==',
|
||||
'compressed.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAADAFBM'
|
||||
'VEX//////8z//5n//2b//zP//wD/zP//zMz/zJn/zGb/zDP/zAD/'
|
||||
'mf//mcz/mZn/mWb/mTP/mQD/Zv//Zsz/Zpn/Zmb/ZjP/ZgD/M///'
|
||||
'M8z/M5n/M2b/MzP/MwD/AP//AMz/AJn/AGb/ADP/AADM///M/8zM'
|
||||
'/5nM/2bM/zPM/wDMzP/MzMzMzJnMzGbMzDPMzADMmf/MmczMmZnM'
|
||||
'mWbMmTPMmQDMZv/MZszMZpnMZmbMZjPMZgDMM//MM8zMM5nMM2bM'
|
||||
'MzPMMwDMAP/MAMzMAJnMAGbMADPMAACZ//+Z/8yZ/5mZ/2aZ/zOZ'
|
||||
'/wCZzP+ZzMyZzJmZzGaZzDOZzACZmf+ZmcyZmZmZmWaZmTOZmQCZ'
|
||||
'Zv+ZZsyZZpmZZmaZZjOZZgCZM/+ZM8yZM5mZM2aZMzOZMwCZAP+Z'
|
||||
'AMyZAJmZAGaZADOZAABm//9m/8xm/5lm/2Zm/zNm/wBmzP9mzMxm'
|
||||
'zJlmzGZmzDNmzABmmf9mmcxmmZlmmWZmmTNmmQBmZv9mZsxmZplm'
|
||||
'ZmZmZjNmZgBmM/9mM8xmM5lmM2ZmMzNmMwBmAP9mAMxmAJlmAGZm'
|
||||
'ADNmAAAz//8z/8wz/5kz/2Yz/zMz/wAzzP8zzMwzzJkzzGYzzDMz'
|
||||
'zAAzmf8zmcwzmZkzmWYzmTMzmQAzZv8zZswzZpkzZmYzZjMzZgAz'
|
||||
'M/8zM8wzM5kzM2YzMzMzMwAzAP8zAMwzAJkzAGYzADMzAAAA//8A'
|
||||
'/8wA/5kA/2YA/zMA/wAAzP8AzMwAzJkAzGYAzDMAzAAAmf8AmcwA'
|
||||
'mZkAmWYAmTMAmQAAZv8AZswAZpkAZmYAZjMAZgAAM/8AM8wAM5kA'
|
||||
'M2YAMzMAMwAAAP8AAMwAAJkAAGYAADPuAADdAAC7AACqAACIAAB3'
|
||||
'AABVAABEAAAiAAARAAAA7gAA3QAAuwAAqgAAiAAAdwAAVQAARAAA'
|
||||
'IgAAEQAAAO4AAN0AALsAAKoAAIgAAHcAAFUAAEQAACIAABHu7u7d'
|
||||
'3d27u7uqqqqIiIh3d3dVVVVEREQiIiIREREAAAD7CIKZAAAAJXRS'
|
||||
'TlP///////////////////////////////////////////////8A'
|
||||
'P89CTwAAAGtJREFUeNp9z9ENgDAIhOEOco+dybVuEXasFMRDY/x5'
|
||||
'+xJCO6Znu6kSx7BhXyjtKBWWNlwW88Loid7hFRKBXiIYCMfMEYUQ'
|
||||
'QohC3CjFA5nIjqx1CqlDLGR/EhM5O06yvin0ftGOyIS7lV14AsQN'
|
||||
'aR7rMEBYAAAAAElFTkSuQmCC',
|
||||
'folder.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAElBMVEX/'
|
||||
'////zJnM//+ZZjMzMzMAAADCEvqoAAAAA3RSTlP//wDXyg1BAAAASElE'
|
||||
'QVR42s3KQQ6AQAhDUaXt/a/sQDrRJu7c+NmQB0e99B3lnqjT6cYx6zSI'
|
||||
'bV40n3D7psYMoBoz4w8/EdNYQsbGEjNxYSljXTEsA9O1pLTvAAAAAElF'
|
||||
'TkSuQmCC',
|
||||
'text.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAD1BMVEX/'
|
||||
'///M//+ZmZkzMzMAAABVsTOVAAAAAnRSTlP/AOW3MEoAAABISURBVHja'
|
||||
'tcrRCgAgCENRbf7/N7dKomGvngjhMsPLD4NdMPwia438NRIyxsaL/XQZ'
|
||||
'hyxpkC6zyjLXGVXnkhqWJWIIrOgeinECLlUCjBCqNQoAAAAASUVORK5C'
|
||||
'YII=',
|
||||
'unknown.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAD1BMVEX/'
|
||||
'///M//+ZmZkzMzMAAABVsTOVAAAAAnRSTlP/AOW3MEoAAABYSURBVHja'
|
||||
'ncvRDoAgDEPRruX/v1kmNHPBxMTLyzgD6FmsILg56g2hQnJkOco4yZhq'
|
||||
'tN5nYd5Zq0LsHblwxwP9GTCWsaGtoelANKzOlz/RfaLYUmLE6E28ALlN'
|
||||
'AupSdoFsAAAAAElFTkSuQmCC'}
|
||||
|
||||
|
||||
# Begin vendored code
|
||||
# This code is licensed under the Public Domain/CC0 and comes from
|
||||
# https://github.com/leenr/gzip-stream/blob/master/gzip_stream.py
|
||||
# Code was modified:
|
||||
# removed type annotations to support python2.
|
||||
# removed use of *, somearg for positional anonymous args.
|
||||
# Default compression level to 9.
|
||||
|
||||
class GZIPCompressedStream(io.RawIOBase):
|
||||
def __init__(self, stream, compression_level=9):
|
||||
assert 1 <= compression_level <= 9
|
||||
|
||||
self._compression_level = compression_level
|
||||
self._stream = stream
|
||||
|
||||
self._compressed_stream = io.BytesIO()
|
||||
self._compressor = gzip.GzipFile(
|
||||
mode='wb',
|
||||
fileobj=self._compressed_stream,
|
||||
compresslevel=compression_level
|
||||
)
|
||||
|
||||
# because of the GZIP header written by `GzipFile.__init__`:
|
||||
self._compressed_stream.seek(0)
|
||||
|
||||
@property
|
||||
def compression_level(self):
|
||||
return self._compression_level
|
||||
|
||||
@property
|
||||
def stream(self):
|
||||
return self._stream
|
||||
|
||||
def readable(self):
|
||||
return True
|
||||
|
||||
def _read_compressed_into(self, b):
|
||||
buf = self._compressed_stream.read(len(b))
|
||||
b[:len(buf)] = buf
|
||||
return len(buf)
|
||||
|
||||
def readinto(self, b):
|
||||
b = memoryview(b)
|
||||
|
||||
offset = 0
|
||||
size = len(b)
|
||||
while offset < size:
|
||||
offset += self._read_compressed_into(b[offset:])
|
||||
if offset < size:
|
||||
# self._compressed_buffer now empty
|
||||
if self._compressor.closed:
|
||||
# nothing to compress anymore
|
||||
break
|
||||
# compress next bytes
|
||||
self._read_n_compress(size)
|
||||
|
||||
return offset
|
||||
|
||||
def _read_n_compress(self, size):
|
||||
assert size > 0
|
||||
|
||||
data = self._stream.read(size)
|
||||
|
||||
# rewind buffer to the start to free up memory
|
||||
# (because anything currently in the buffer should be already
|
||||
# streamed off the object)
|
||||
self._compressed_stream.seek(0)
|
||||
self._compressed_stream.truncate(0)
|
||||
|
||||
if data:
|
||||
self._compressor.write(data)
|
||||
else:
|
||||
# this will write final data (will flush zlib with Z_FINISH)
|
||||
self._compressor.close()
|
||||
|
||||
# rewind to the buffer start
|
||||
self._compressed_stream.seek(0)
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
'{self.__class__.__name__}('
|
||||
'{self.stream!r}, '
|
||||
'compression_level={self.compression_level!r}'
|
||||
')'
|
||||
).format(self=self)
|
||||
|
||||
# End vendored code
|
||||
|
||||
|
||||
def get_mime_icon(mime, filename=''):
|
||||
icon = (APACHE_FILE_ICON_MAP.get(filename) or
|
||||
APACHE_MIME_ICON_MAP.get(mime) or
|
||||
APACHE_MIME_ICON_MAP['_default'])
|
||||
return "data:image/png;base64,%s" % ICON_IMAGES[icon]
|
||||
|
||||
|
||||
def get_cloud(cloud):
|
||||
if isinstance(cloud, dict):
|
||||
config = openstack.config.loader.OpenStackConfig().get_one(**cloud)
|
||||
return openstack.connection.Connection(config=config)
|
||||
else:
|
||||
return openstack.connect(cloud=cloud)
|
||||
|
||||
|
||||
def retry_function(func):
|
||||
for attempt in range(1, POST_ATTEMPTS + 1):
|
||||
try:
|
||||
return func()
|
||||
except Exception:
|
||||
if attempt >= POST_ATTEMPTS:
|
||||
raise
|
||||
else:
|
||||
logging.exception("Error on attempt %d" % attempt)
|
||||
time.sleep(attempt * 10)
|
||||
|
||||
|
||||
def sizeof_fmt(num, suffix='B'):
|
||||
# From http://stackoverflow.com/questions/1094841/
|
||||
# reusable-library-to-get-human-readable-version-of-file-size
|
||||
for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
|
||||
if abs(num) < 1024.0:
|
||||
return "%3.1f%s%s" % (num, unit, suffix)
|
||||
num /= 1024.0
|
||||
return "%.1f%s%s" % (num, 'Y', suffix)
|
||||
|
||||
|
||||
class FileDetail():
|
||||
"""
|
||||
Used to generate indexes with links or as the file path
|
||||
to push to swift.
|
||||
"""
|
||||
|
||||
def __init__(self, full_path, relative_path, filename=None):
|
||||
"""
|
||||
Args:
|
||||
full_path (str): The absolute path to the file on disk.
|
||||
relative_path (str): The relative path from the artifacts source
|
||||
used for links.
|
||||
filename (str): An optional alternate filename in links.
|
||||
"""
|
||||
# Make FileNotFoundError exception to be compatible with python2
|
||||
try:
|
||||
FileNotFoundError # noqa: F823
|
||||
except NameError:
|
||||
FileNotFoundError = OSError
|
||||
|
||||
self.full_path = full_path
|
||||
if filename is None:
|
||||
self.filename = os.path.basename(full_path)
|
||||
else:
|
||||
self.filename = filename
|
||||
self.relative_path = relative_path
|
||||
|
||||
if self.full_path and os.path.isfile(self.full_path):
|
||||
mime_guess, encoding = mimetypes.guess_type(self.full_path)
|
||||
self.mimetype = mime_guess if mime_guess else 'text/plain'
|
||||
self.encoding = encoding
|
||||
self.folder = False
|
||||
else:
|
||||
self.mimetype = 'application/directory'
|
||||
self.encoding = None
|
||||
self.folder = True
|
||||
try:
|
||||
st = os.stat(self.full_path)
|
||||
self.last_modified = time.gmtime(st[stat.ST_MTIME])
|
||||
self.size = st[stat.ST_SIZE]
|
||||
except (FileNotFoundError, TypeError):
|
||||
self.last_modified = time.gmtime(0)
|
||||
self.size = 0
|
||||
|
||||
def __repr__(self):
|
||||
t = 'Folder' if self.folder else 'File'
|
||||
return '<%s %s>' % (t, self.relative_path)
|
||||
|
||||
|
||||
class FileList(Sequence):
|
||||
'''A collection of FileDetail objects
|
||||
|
||||
This is a list-like group of FileDetail objects, intended to be
|
||||
used as a context manager around the upload process.
|
||||
'''
|
||||
def __init__(self):
|
||||
self.file_list = []
|
||||
self.file_list.append(FileDetail(None, '', ''))
|
||||
self.tempdirs = []
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, type, value, traceback):
|
||||
for tempdir in self.tempdirs:
|
||||
shutil.rmtree(tempdir)
|
||||
|
||||
def __getitem__(self, item):
|
||||
return self.file_list.__getitem__(item)
|
||||
|
||||
def __len__(self):
|
||||
return self.file_list.__len__()
|
||||
|
||||
def get_tempdir(self):
|
||||
'''Get a temporary directory
|
||||
|
||||
Returns path to a private temporary directory which will be
|
||||
cleaned on exit
|
||||
'''
|
||||
tempdir = tempfile.mkdtemp(prefix='s-u-l-tmp')
|
||||
self.tempdirs.append(tempdir)
|
||||
return tempdir
|
||||
|
||||
@staticmethod
|
||||
def _path_in_tree(root, path):
|
||||
full_path = os.path.realpath(os.path.abspath(
|
||||
os.path.expanduser(path)))
|
||||
if not full_path.startswith(root):
|
||||
logging.debug("Skipping path outside root: %s" % (path,))
|
||||
return False
|
||||
return True
|
||||
|
||||
def add(self, file_path):
|
||||
"""
|
||||
Generate a list of files to upload to swift. Recurses through
|
||||
directories
|
||||
"""
|
||||
|
||||
# file_list: A list of FileDetails to push to swift
|
||||
file_list = []
|
||||
|
||||
if os.path.isfile(file_path):
|
||||
relative_path = os.path.basename(file_path)
|
||||
file_list.append(FileDetail(file_path, relative_path))
|
||||
elif os.path.isdir(file_path):
|
||||
original_root = os.path.realpath(os.path.abspath(
|
||||
os.path.expanduser(file_path)))
|
||||
|
||||
parent_dir = os.path.dirname(file_path)
|
||||
if not file_path.endswith('/'):
|
||||
filename = os.path.basename(file_path)
|
||||
full_path = file_path
|
||||
relative_name = os.path.relpath(full_path, parent_dir)
|
||||
file_list.append(FileDetail(full_path, relative_name,
|
||||
filename))
|
||||
# TODO: this will copy the result of symlinked files, but
|
||||
# it won't follow directory symlinks. If we add that, we
|
||||
# should ensure that we don't loop.
|
||||
for path, folders, files in os.walk(file_path):
|
||||
# Sort folder in-place so that we recurse in order.
|
||||
files.sort(key=lambda x: x.lower())
|
||||
folders.sort(key=lambda x: x.lower())
|
||||
# relative_path: The path between the given directory
|
||||
# and the one being currently walked.
|
||||
relative_path = os.path.relpath(path, parent_dir)
|
||||
|
||||
for filename in folders:
|
||||
full_path = os.path.join(path, filename)
|
||||
if not self._path_in_tree(original_root, full_path):
|
||||
continue
|
||||
relative_name = os.path.relpath(full_path, parent_dir)
|
||||
file_list.append(FileDetail(full_path, relative_name,
|
||||
filename))
|
||||
|
||||
for filename in files:
|
||||
full_path = os.path.join(path, filename)
|
||||
if not self._path_in_tree(original_root, full_path):
|
||||
continue
|
||||
relative_name = os.path.relpath(full_path, parent_dir)
|
||||
file_detail = FileDetail(full_path, relative_name)
|
||||
file_list.append(file_detail)
|
||||
|
||||
self.file_list += file_list
|
||||
|
||||
|
||||
class Indexer():
|
||||
"""Index a FileList
|
||||
|
||||
Functions to generate indexes and other collated data for a
|
||||
FileList
|
||||
|
||||
- make_indexes() : make index.html in folders
|
||||
"""
|
||||
def __init__(self, file_list):
|
||||
'''
|
||||
Args:
|
||||
file_list (FileList): A FileList object with all files
|
||||
to be indexed.
|
||||
'''
|
||||
assert isinstance(file_list, FileList)
|
||||
self.file_list = file_list
|
||||
|
||||
def _make_index_file(self, folder_links, title, tempdir, append_footer):
|
||||
"""Writes an index into a file for pushing"""
|
||||
for file_details in folder_links:
|
||||
# Do not generate an index file if one exists already.
|
||||
# This may be the case when uploading other machine generated
|
||||
# content like python coverage info.
|
||||
if self.index_filename == file_details.filename:
|
||||
return
|
||||
index_content = self._generate_log_index(
|
||||
folder_links, title, append_footer)
|
||||
fd = open(os.path.join(tempdir, self.index_filename), 'w')
|
||||
fd.write(index_content)
|
||||
return os.path.join(tempdir, self.index_filename)
|
||||
|
||||
def _generate_log_index(self, folder_links, title, append_footer):
|
||||
"""Create an index of logfiles and links to them"""
|
||||
|
||||
output = '<html><head><title>%s</title></head><body>\n' % title
|
||||
output += '<h1>%s</h1>\n' % title
|
||||
output += '<table><tr><th></th><th>Name</th><th>Last Modified</th>'
|
||||
output += '<th>Size</th></tr>'
|
||||
|
||||
file_details_to_append = None
|
||||
for file_details in folder_links:
|
||||
output += '<tr>'
|
||||
output += (
|
||||
'<td><img alt="[ ]" title="%(m)s" src="%(i)s"></img></td>' % ({
|
||||
'm': file_details.mimetype,
|
||||
'i': get_mime_icon(file_details.mimetype,
|
||||
file_details.filename),
|
||||
}))
|
||||
filename = file_details.filename
|
||||
if file_details.folder:
|
||||
filename += '/'
|
||||
output += '<td><a href="%s">%s</a></td>' % (
|
||||
urlparse.quote(filename),
|
||||
filename)
|
||||
output += '<td>%s</td>' % time.asctime(
|
||||
file_details.last_modified)
|
||||
size = sizeof_fmt(file_details.size, suffix='')
|
||||
output += '<td style="text-align: right">%s</td>' % size
|
||||
output += '</tr>\n'
|
||||
|
||||
if (append_footer and
|
||||
append_footer in file_details.filename):
|
||||
file_details_to_append = file_details
|
||||
|
||||
output += '</table>'
|
||||
|
||||
if file_details_to_append:
|
||||
output += '<br /><hr />'
|
||||
try:
|
||||
with open(file_details_to_append.full_path, 'r') as f:
|
||||
output += f.read()
|
||||
except IOError:
|
||||
logging.exception("Error opening file for appending")
|
||||
|
||||
output += '</body></html>\n'
|
||||
return output
|
||||
|
||||
def make_indexes(self, create_parent_links=True,
|
||||
create_topdir_parent_link=False,
|
||||
append_footer='index_footer.html'):
|
||||
'''Make index.html files
|
||||
|
||||
Iterate the file list and crete index.html files for folders
|
||||
|
||||
Args:
|
||||
create_parent_links (bool): Create parent links
|
||||
create_topdir_parent_link (bool): Create topdir parent link
|
||||
append_footer (str): Filename of a footer to append to each
|
||||
generated page
|
||||
|
||||
Return:
|
||||
No value, the self.file_list will be updated
|
||||
'''
|
||||
self.index_filename = 'index.html'
|
||||
|
||||
folders = collections.OrderedDict()
|
||||
for f in self.file_list:
|
||||
if f.folder:
|
||||
folders[f.relative_path] = []
|
||||
folder = os.path.dirname(os.path.dirname(
|
||||
f.relative_path + '/'))
|
||||
if folder == '/':
|
||||
folder = ''
|
||||
else:
|
||||
folder = os.path.dirname(f.relative_path)
|
||||
folders[folder].append(f)
|
||||
|
||||
indexes = {}
|
||||
parent_file_detail = FileDetail(None, '..', '..')
|
||||
for folder, files in folders.items():
|
||||
# Don't add the pseudo-top-directory
|
||||
if files and files[0].full_path is None:
|
||||
files = files[1:]
|
||||
if create_topdir_parent_link:
|
||||
files = [parent_file_detail] + files
|
||||
elif create_parent_links:
|
||||
files = [parent_file_detail] + files
|
||||
|
||||
# Do generate a link to the parent directory
|
||||
full_path = self._make_index_file(files, 'Index of %s' % (folder,),
|
||||
self.file_list.get_tempdir(),
|
||||
append_footer)
|
||||
|
||||
if full_path:
|
||||
filename = os.path.basename(full_path)
|
||||
relative_name = os.path.join(folder, filename)
|
||||
indexes[folder] = FileDetail(full_path, relative_name)
|
||||
|
||||
# This appends the index file at the end of the group of files
|
||||
# for each directory.
|
||||
new_list = []
|
||||
last_dirname = None
|
||||
for f in reversed(list(self.file_list)):
|
||||
if f.folder:
|
||||
relative_path = f.relative_path + '/'
|
||||
else:
|
||||
relative_path = f.relative_path
|
||||
dirname = os.path.dirname(relative_path)
|
||||
if dirname == '/':
|
||||
dirname = ''
|
||||
if dirname != last_dirname:
|
||||
index = indexes.pop(dirname, None)
|
||||
if index:
|
||||
new_list.append(index)
|
||||
last_dirname = dirname
|
||||
new_list.append(f)
|
||||
new_list.reverse()
|
||||
self.file_list.file_list = new_list
|
||||
|
||||
|
||||
class GzipFilter():
|
||||
chunk_size = 16384
|
||||
|
||||
def __init__(self, infile):
|
||||
self.gzipfile = GZIPCompressedStream(infile)
|
||||
self.done = False
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
if self.done:
|
||||
self.gzipfile.close()
|
||||
raise StopIteration()
|
||||
data = self.gzipfile.read(self.chunk_size)
|
||||
if not data:
|
||||
self.done = True
|
||||
return data
|
||||
|
||||
|
||||
class DeflateFilter():
|
||||
chunk_size = 16384
|
||||
|
||||
def __init__(self, infile):
|
||||
self.infile = infile
|
||||
self.encoder = zlib.compressobj()
|
||||
self.done = False
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
if self.done:
|
||||
raise StopIteration()
|
||||
ret = b''
|
||||
while True:
|
||||
data = self.infile.read(self.chunk_size)
|
||||
if data:
|
||||
ret = self.encoder.compress(data)
|
||||
if ret:
|
||||
break
|
||||
else:
|
||||
self.done = True
|
||||
ret = self.encoder.flush()
|
||||
break
|
||||
return ret
|
||||
|
||||
|
||||
class Uploader():
|
||||
def __init__(self, cloud, container, prefix=None, delete_after=None,
|
||||
public=True):
|
||||
self.cloud = cloud
|
||||
self.container = container
|
||||
self.prefix = prefix or ''
|
||||
self.delete_after = delete_after
|
||||
|
||||
sess = self.cloud.config.get_session()
|
||||
adapter = requests.adapters.HTTPAdapter(pool_maxsize=100)
|
||||
sess.mount('https://', adapter)
|
||||
|
||||
# If we're in Rackspace, there's some non-standard stuff we
|
||||
# need to do to get the public endpoint.
|
||||
try:
|
||||
cdn_endpoint = self.cloud.session.auth.get_endpoint(
|
||||
self.cloud.session, service_type='rax:object-cdn',
|
||||
region_name=self.cloud.config.region_name,
|
||||
interface=self.cloud.config.interface)
|
||||
cdn_url = os.path.join(cdn_endpoint, self.container)
|
||||
except keystoneauth1.exceptions.catalog.EndpointNotFound:
|
||||
cdn_url = None
|
||||
|
||||
# We retry here because sometimes we get HTTP 401 errors in rax.
|
||||
# They seem to happen infrequently (on the order of once a day across
|
||||
# all jobs) so a retry is likely to work.
|
||||
container = retry_function(
|
||||
lambda: self.cloud.get_container(self.container))
|
||||
if not container:
|
||||
retry_function(
|
||||
lambda: self.cloud.create_container(
|
||||
name=self.container, public=public))
|
||||
headers = {'X-Container-Meta-Web-Index': 'index.html',
|
||||
'X-Container-Meta-Access-Control-Allow-Origin': '*'}
|
||||
retry_function(
|
||||
lambda: self.cloud.update_container(
|
||||
name=self.container,
|
||||
headers=headers))
|
||||
# 'X-Container-Meta-Web-Listings': 'true'
|
||||
|
||||
# The ceph radosgw swift implementation requires an
|
||||
# index.html at the root in order for any other indexes to
|
||||
# work.
|
||||
index_headers = {'access-control-allow-origin': '*'}
|
||||
retry_function(
|
||||
lambda: self.cloud.create_object(self.container,
|
||||
name='index.html',
|
||||
data='',
|
||||
content_type='text/html',
|
||||
**index_headers))
|
||||
|
||||
# Enable the CDN in rax
|
||||
if cdn_url:
|
||||
retry_function(lambda: self.cloud.session.put(cdn_url))
|
||||
|
||||
if cdn_url:
|
||||
endpoint = retry_function(
|
||||
lambda: self.cloud.session.head(
|
||||
cdn_url).headers['X-Cdn-Ssl-Uri'])
|
||||
container = endpoint
|
||||
else:
|
||||
endpoint = self.cloud.object_store.get_endpoint()
|
||||
container = os.path.join(endpoint, self.container)
|
||||
|
||||
self.url = os.path.join(container, self.prefix)
|
||||
|
||||
def upload(self, file_list):
|
||||
"""Spin up thread pool to upload to swift"""
|
||||
num_threads = min(len(file_list), MAX_UPLOAD_THREADS)
|
||||
threads = []
|
||||
queue = queuelib.Queue()
|
||||
# add items to queue
|
||||
for f in file_list:
|
||||
queue.put(f)
|
||||
|
||||
for x in range(num_threads):
|
||||
t = threading.Thread(target=self.post_thread, args=(queue,))
|
||||
threads.append(t)
|
||||
t.start()
|
||||
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
def post_thread(self, queue):
|
||||
while True:
|
||||
try:
|
||||
file_detail = queue.get_nowait()
|
||||
logging.debug("%s: processing job %s",
|
||||
threading.current_thread(),
|
||||
file_detail)
|
||||
retry_function(lambda: self._post_file(file_detail))
|
||||
except requests.exceptions.RequestException:
|
||||
# Do our best to attempt to upload all the files
|
||||
logging.exception("Error posting file after multiple attempts")
|
||||
continue
|
||||
except IOError:
|
||||
# Do our best to attempt to upload all the files
|
||||
logging.exception("Error opening file")
|
||||
continue
|
||||
except queuelib.Empty:
|
||||
# No more work to do
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def _is_text_type(mimetype):
|
||||
# We want to compress all text types.
|
||||
if mimetype.startswith('text/'):
|
||||
return True
|
||||
|
||||
# Further compress types that typically contain text but are no
|
||||
# text sub type.
|
||||
compress_types = [
|
||||
'application/json',
|
||||
'image/svg+xml',
|
||||
]
|
||||
if mimetype in compress_types:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _post_file(self, file_detail):
|
||||
relative_path = os.path.join(self.prefix, file_detail.relative_path)
|
||||
headers = {}
|
||||
if self.delete_after:
|
||||
headers['x-delete-after'] = str(self.delete_after)
|
||||
headers['content-type'] = file_detail.mimetype
|
||||
# This is required for Rackspace CDN
|
||||
headers['access-control-allow-origin'] = '*'
|
||||
|
||||
if not file_detail.folder:
|
||||
if (file_detail.encoding is None and
|
||||
self._is_text_type(file_detail.mimetype)):
|
||||
headers['content-encoding'] = 'gzip'
|
||||
data = GzipFilter(open(file_detail.full_path, 'rb'))
|
||||
else:
|
||||
if file_detail.encoding:
|
||||
headers['content-encoding'] = file_detail.encoding
|
||||
data = open(file_detail.full_path, 'rb')
|
||||
else:
|
||||
data = ''
|
||||
relative_path = relative_path.rstrip('/')
|
||||
if relative_path == '':
|
||||
relative_path = '/'
|
||||
self.cloud.create_object(self.container,
|
||||
name=relative_path,
|
||||
data=data,
|
||||
**headers)
|
||||
|
||||
|
||||
def run(cloud, container, files,
|
||||
indexes=True, parent_links=True, topdir_parent_link=False,
|
||||
partition=False, footer='index_footer.html', delete_after=15552000,
|
||||
prefix=None, public=True, dry_run=False):
|
||||
|
||||
if prefix:
|
||||
prefix = prefix.lstrip('/')
|
||||
if partition and prefix:
|
||||
parts = prefix.split('/')
|
||||
if len(parts) > 1:
|
||||
container += '_' + parts[0]
|
||||
prefix = '/'.join(parts[1:])
|
||||
|
||||
# Create the objects to make sure the arguments are sound.
|
||||
with FileList() as file_list:
|
||||
# Scan the files.
|
||||
for file_path in files:
|
||||
file_list.add(file_path)
|
||||
|
||||
indexer = Indexer(file_list)
|
||||
|
||||
# (Possibly) make indexes.
|
||||
if indexes:
|
||||
indexer.make_indexes(create_parent_links=parent_links,
|
||||
create_topdir_parent_link=topdir_parent_link,
|
||||
append_footer=footer)
|
||||
|
||||
logging.debug("List of files prepared to upload:")
|
||||
for x in file_list:
|
||||
logging.debug(x)
|
||||
|
||||
# Do no connect to swift or do any uploading in a dry run
|
||||
if dry_run:
|
||||
# No URL is known, so return nothing
|
||||
return
|
||||
|
||||
# Upload.
|
||||
uploader = Uploader(cloud, container, prefix, delete_after,
|
||||
public)
|
||||
uploader.upload(file_list)
|
||||
return uploader.url
|
||||
|
||||
|
||||
def ansible_main():
|
||||
module = AnsibleModule(
|
||||
argument_spec=dict(
|
||||
cloud=dict(required=True, type='raw'),
|
||||
container=dict(required=True, type='str'),
|
||||
files=dict(required=True, type='list'),
|
||||
partition=dict(type='bool', default=False),
|
||||
indexes=dict(type='bool', default=True),
|
||||
parent_links=dict(type='bool', default=True),
|
||||
topdir_parent_link=dict(type='bool', default=False),
|
||||
public=dict(type='bool', default=True),
|
||||
footer=dict(type='str'),
|
||||
delete_after=dict(type='int'),
|
||||
prefix=dict(type='str'),
|
||||
)
|
||||
)
|
||||
|
||||
p = module.params
|
||||
cloud = get_cloud(p.get('cloud'))
|
||||
try:
|
||||
url = run(cloud, p.get('container'), p.get('files'),
|
||||
indexes=p.get('indexes'),
|
||||
parent_links=p.get('parent_links'),
|
||||
topdir_parent_link=p.get('topdir_parent_link'),
|
||||
partition=p.get('partition'),
|
||||
footer=p.get('footer'),
|
||||
delete_after=p.get('delete_after', 15552000),
|
||||
prefix=p.get('prefix'),
|
||||
public=p.get('public'))
|
||||
except (keystoneauth1.exceptions.http.HttpError,
|
||||
requests.exceptions.RequestException):
|
||||
s = "Error uploading to %s.%s" % (cloud.name, cloud.config.region_name)
|
||||
logging.exception(s)
|
||||
s += "\n" + traceback.format_exc()
|
||||
module.fail_json(
|
||||
changed=False,
|
||||
msg=s,
|
||||
cloud=cloud.name,
|
||||
region_name=cloud.config.region_name)
|
||||
module.exit_json(changed=True,
|
||||
url=url)
|
||||
|
||||
|
||||
def cli_main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Upload files to swift"
|
||||
)
|
||||
parser.add_argument('--verbose', action='store_true',
|
||||
help='show debug information')
|
||||
parser.add_argument('--no-indexes', action='store_true',
|
||||
help='do not generate any indexes at all')
|
||||
parser.add_argument('--no-parent-links', action='store_true',
|
||||
help='do not include links back to a parent dir')
|
||||
parser.add_argument('--create-topdir-parent-link', action='store_true',
|
||||
help='include a link in the root directory of the '
|
||||
'files to the parent directory which may be the '
|
||||
'index of all results')
|
||||
parser.add_argument('--no-public', action='store_true',
|
||||
help='do not create the container as public')
|
||||
parser.add_argument('--partition', action='store_true',
|
||||
help='partition the prefix into multiple containers')
|
||||
parser.add_argument('--append-footer', default='index_footer.html',
|
||||
help='when generating an index, if the given file is '
|
||||
'present in a directory, append it to the index '
|
||||
'(set to "none" to disable)')
|
||||
parser.add_argument('--delete-after', default=15552000,
|
||||
help='Number of seconds to delete object after '
|
||||
'upload. Default is 6 months (15552000 seconds) '
|
||||
'and if set to 0 X-Delete-After will not be set',
|
||||
type=int)
|
||||
parser.add_argument('--prefix',
|
||||
help='Prepend this path to the object names when '
|
||||
'uploading')
|
||||
parser.add_argument('--dry-run', action='store_true',
|
||||
help='do not attempt to create containers or upload, '
|
||||
'useful with --verbose for debugging')
|
||||
parser.add_argument('cloud',
|
||||
help='Name of the cloud to use when uploading')
|
||||
parser.add_argument('container',
|
||||
help='Name of the container to use when uploading')
|
||||
parser.add_argument('files', nargs='+',
|
||||
help='the file(s) to upload with recursive glob '
|
||||
'matching when supplied as a string')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.verbose:
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
# Set requests log level accordingly
|
||||
logging.getLogger("requests").setLevel(logging.DEBUG)
|
||||
# logging.getLogger("keystoneauth").setLevel(logging.INFO)
|
||||
# logging.getLogger("stevedore").setLevel(logging.INFO)
|
||||
logging.captureWarnings(True)
|
||||
|
||||
append_footer = args.append_footer
|
||||
if append_footer.lower() == 'none':
|
||||
append_footer = None
|
||||
|
||||
url = run(get_cloud(args.cloud), args.container, args.files,
|
||||
indexes=not args.no_indexes,
|
||||
parent_links=not args.no_parent_links,
|
||||
topdir_parent_link=args.create_topdir_parent_link,
|
||||
partition=args.partition,
|
||||
footer=append_footer,
|
||||
delete_after=args.delete_after,
|
||||
prefix=args.prefix,
|
||||
public=not args.no_public,
|
||||
dry_run=args.dry_run)
|
||||
print(url)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Avoid unactionable warnings
|
||||
requestsexceptions.squelch_warnings(
|
||||
requestsexceptions.InsecureRequestWarning)
|
||||
|
||||
if sys.stdin.isatty():
|
||||
cli_main()
|
||||
else:
|
||||
ansible_main()
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
- name: Set zuul-log-path fact
|
||||
include_role:
|
||||
name: set-zuul-log-path-fact
|
||||
when: zuul_log_path is not defined
|
||||
|
||||
# Always upload (true), never upload (false) or only on failure ('failure')
|
||||
- when: zuul_site_upload_logs | default(true) | bool or
|
||||
(zuul_site_upload_logs == 'failure' and not zuul_success | bool)
|
||||
block:
|
||||
# Use chmod instead of file because ansible 2.5 file with recurse and
|
||||
# follow can't really handle symlinks to .
|
||||
- name: Ensure logs are readable before uploading
|
||||
delegate_to: localhost
|
||||
command: "chmod -R u=rwX,g=rX,o=rX {{ zuul.executor.log_root }}/"
|
||||
# ANSIBLE0007 chmod used in place of argument mode to file
|
||||
tags:
|
||||
- skip_ansible_lint
|
||||
|
||||
- name: Upload logs to swift
|
||||
delegate_to: localhost
|
||||
zuul_swift_upload:
|
||||
cloud: "{{ zuul_log_cloud_config }}"
|
||||
partition: "{{ zuul_log_partition }}"
|
||||
container: "{{ zuul_log_container }}"
|
||||
public: "{{ zuul_log_container_public }}"
|
||||
prefix: "{{ zuul_log_path }}"
|
||||
indexes: "{{ zuul_log_create_indexes }}"
|
||||
files:
|
||||
- "{{ zuul.executor.log_root }}/"
|
||||
delete_after: "{{ zuul_log_delete_after | default(omit) }}"
|
||||
register: upload_results
|
||||
|
||||
- name: Return log URL to Zuul
|
||||
delegate_to: localhost
|
||||
zuul_return:
|
||||
data:
|
||||
zuul:
|
||||
log_url: "{{ upload_results.url }}/"
|
||||
when: upload_results is defined
|
||||
Loading…
Reference in New Issue