diff --git a/roles/upload-logs-base/library/test_index.py b/roles/upload-logs-base/library/test_index.py new file mode 100644 index 000000000..69832d473 --- /dev/null +++ b/roles/upload-logs-base/library/test_index.py @@ -0,0 +1,481 @@ +# Copyright (C) 2018 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import six +import testtools +import time +import stat +import fixtures +try: + from unittest import mock +except ImportError: + import mock + +import requests +from bs4 import BeautifulSoup +from .zuul_swift_upload import Uploader +from ..module_utils.zuul_jobs.upload_utils import FileList, Indexer, FileDetail + +FIXTURE_DIR = os.path.join(os.path.dirname(__file__), + 'test-fixtures') + + +class SymlinkFixture(fixtures.Fixture): + links = [ + ('bad_symlink', '/etc'), + ('bad_symlink_file', '/etc/issue'), + ('good_symlink', 'controller'), + ('recursive_symlink', '.'), + ('symlink_file', 'job-output.json'), + ('symlink_loop_a', 'symlink_loop'), + ('symlink_loop/symlink_loop_b', '..'), + ] + + def _setUp(self): + for (src, target) in self.links: + path = os.path.join(FIXTURE_DIR, 'links', src) + os.symlink(target, path) + self.addCleanup(os.unlink, path) + + +class TestFileList(testtools.TestCase): + + def assert_files(self, result, files): + self.assertEqual(len(result), len(files)) + for expected, received in zip(files, result): + e = expected[0] + if six.PY2: + e = e.encode('utf-8') + self.assertEqual(e, received.relative_path) + if e and e[0][-1] == '/': + efilename = os.path.split( + os.path.dirname(e))[1] + '/' + else: + efilename = os.path.split(e)[1] + self.assertEqual(efilename, received.filename) + if received.folder: + if received.full_path is not None and expected[0] != '': + self.assertTrue(os.path.isdir(received.full_path)) + else: + self.assertTrue(os.path.isfile(received.full_path)) + self.assertEqual(expected[1], received.mimetype) + self.assertEqual(expected[2], received.encoding) + + def find_file(self, file_list, path): + for f in file_list: + if f.relative_path == path: + return f + + def test_single_dir_trailing_slash(self): + '''Test a single directory with a trailing slash''' + + with FileList() as fl: + fl.add(os.path.join(FIXTURE_DIR, 'logs/')) + self.assert_files(fl, [ + ('', 'application/directory', None), + ('controller', 'application/directory', None), + ('zuul-info', 'application/directory', None), + ('job-output.json', 'application/json', None), + (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', + 'text/plain', None), + ('controller/subdir', 'application/directory', None), + ('controller/compressed.gz', 'text/plain', 'gzip'), + ('controller/cpu-load.svg', 'image/svg+xml', None), + ('controller/journal.xz', 'text/plain', 'xz'), + ('controller/service_log.txt', 'text/plain', None), + ('controller/syslog', 'text/plain', None), + ('controller/subdir/foo::3.txt', 'text/plain', None), + ('controller/subdir/subdir.txt', 'text/plain', None), + ('zuul-info/inventory.yaml', 'text/plain', None), + ('zuul-info/zuul-info.controller.txt', 'text/plain', None), + ]) + + def test_single_dir(self): + '''Test a single directory without a trailing slash''' + with FileList() as fl: + fl.add(os.path.join(FIXTURE_DIR, 'logs')) + self.assert_files(fl, [ + ('', 'application/directory', None), + ('logs', 'application/directory', None), + ('logs/controller', 'application/directory', None), + ('logs/zuul-info', 'application/directory', None), + ('logs/job-output.json', 'application/json', None), + (u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt', + 'text/plain', None), + ('logs/controller/subdir', 'application/directory', None), + ('logs/controller/compressed.gz', 'text/plain', 'gzip'), + ('logs/controller/cpu-load.svg', 'image/svg+xml', None), + ('logs/controller/journal.xz', 'text/plain', 'xz'), + ('logs/controller/service_log.txt', 'text/plain', None), + ('logs/controller/syslog', 'text/plain', None), + ('logs/controller/subdir/foo::3.txt', 'text/plain', None), + ('logs/controller/subdir/subdir.txt', 'text/plain', None), + ('logs/zuul-info/inventory.yaml', 'text/plain', None), + ('logs/zuul-info/zuul-info.controller.txt', + 'text/plain', None), + ]) + + def test_single_file(self): + '''Test a single file''' + with FileList() as fl: + fl.add(os.path.join(FIXTURE_DIR, + 'logs/zuul-info/inventory.yaml')) + self.assert_files(fl, [ + ('', 'application/directory', None), + ('inventory.yaml', 'text/plain', None), + ]) + + def test_symlinks(self): + '''Test symlinks''' + with FileList() as fl: + self.useFixture(SymlinkFixture()) + fl.add(os.path.join(FIXTURE_DIR, 'links/')) + self.assert_files(fl, [ + ('', 'application/directory', None), + ('controller', 'application/directory', None), + ('good_symlink', 'application/directory', None), + ('recursive_symlink', 'application/directory', None), + ('symlink_loop', 'application/directory', None), + ('symlink_loop_a', 'application/directory', None), + ('job-output.json', 'application/json', None), + ('symlink_file', 'text/plain', None), + ('controller/service_log.txt', 'text/plain', None), + ('symlink_loop/symlink_loop_b', 'application/directory', None), + ('symlink_loop/placeholder', 'text/plain', None), + ]) + + def test_index_files(self): + '''Test index generation''' + with FileList() as fl: + fl.add(os.path.join(FIXTURE_DIR, 'logs')) + ix = Indexer(fl) + ix.make_indexes() + + self.assert_files(fl, [ + ('', 'application/directory', None), + ('index.html', 'text/html', None), + ('logs', 'application/directory', None), + ('logs/controller', 'application/directory', None), + ('logs/zuul-info', 'application/directory', None), + ('logs/job-output.json', 'application/json', None), + (u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt', + 'text/plain', None), + ('logs/index.html', 'text/html', None), + ('logs/controller/subdir', 'application/directory', None), + ('logs/controller/compressed.gz', 'text/plain', 'gzip'), + ('logs/controller/cpu-load.svg', 'image/svg+xml', None), + ('logs/controller/journal.xz', 'text/plain', 'xz'), + ('logs/controller/service_log.txt', 'text/plain', None), + ('logs/controller/syslog', 'text/plain', None), + ('logs/controller/index.html', 'text/html', None), + ('logs/controller/subdir/foo::3.txt', 'text/plain', None), + ('logs/controller/subdir/subdir.txt', 'text/plain', None), + ('logs/controller/subdir/index.html', 'text/html', None), + ('logs/zuul-info/inventory.yaml', 'text/plain', None), + ('logs/zuul-info/zuul-info.controller.txt', + 'text/plain', None), + ('logs/zuul-info/index.html', 'text/html', None), + ]) + + top_index = self.find_file(fl, 'index.html') + page = open(top_index.full_path).read() + page = BeautifulSoup(page, 'html.parser') + rows = page.find_all('tr')[1:] + + self.assertEqual(len(rows), 1) + + self.assertEqual(rows[0].find('a').get('href'), 'logs/index.html') + self.assertEqual(rows[0].find('a').text, 'logs/') + + subdir_index = self.find_file( + fl, 'logs/controller/subdir/index.html') + page = open(subdir_index.full_path).read() + page = BeautifulSoup(page, 'html.parser') + rows = page.find_all('tr')[1:] + self.assertEqual(rows[0].find('a').get('href'), '../index.html') + self.assertEqual(rows[0].find('a').text, '../') + + # Test proper escaping of files with funny names + self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt') + self.assertEqual(rows[1].find('a').text, 'foo::3.txt') + # Test files without escaping + self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt') + self.assertEqual(rows[2].find('a').text, 'subdir.txt') + + def test_index_files_trailing_slash(self): + '''Test index generation with a trailing slash''' + with FileList() as fl: + fl.add(os.path.join(FIXTURE_DIR, 'logs/')) + ix = Indexer(fl) + ix.make_indexes() + + self.assert_files(fl, [ + ('', 'application/directory', None), + ('controller', 'application/directory', None), + ('zuul-info', 'application/directory', None), + ('job-output.json', 'application/json', None), + (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', + 'text/plain', None), + ('index.html', 'text/html', None), + ('controller/subdir', 'application/directory', None), + ('controller/compressed.gz', 'text/plain', 'gzip'), + ('controller/cpu-load.svg', 'image/svg+xml', None), + ('controller/journal.xz', 'text/plain', 'xz'), + ('controller/service_log.txt', 'text/plain', None), + ('controller/syslog', 'text/plain', None), + ('controller/index.html', 'text/html', None), + ('controller/subdir/foo::3.txt', 'text/plain', None), + ('controller/subdir/subdir.txt', 'text/plain', None), + ('controller/subdir/index.html', 'text/html', None), + ('zuul-info/inventory.yaml', 'text/plain', None), + ('zuul-info/zuul-info.controller.txt', 'text/plain', None), + ('zuul-info/index.html', 'text/html', None), + ]) + + top_index = self.find_file(fl, 'index.html') + page = open(top_index.full_path).read() + page = BeautifulSoup(page, 'html.parser') + rows = page.find_all('tr')[1:] + + self.assertEqual(len(rows), 4) + + self.assertEqual(rows[0].find('a').get('href'), + 'controller/index.html') + self.assertEqual(rows[0].find('a').text, 'controller/') + + self.assertEqual(rows[1].find('a').get('href'), + 'zuul-info/index.html') + self.assertEqual(rows[1].find('a').text, 'zuul-info/') + + subdir_index = self.find_file(fl, 'controller/subdir/index.html') + page = open(subdir_index.full_path).read() + page = BeautifulSoup(page, 'html.parser') + rows = page.find_all('tr')[1:] + self.assertEqual(rows[0].find('a').get('href'), '../index.html') + self.assertEqual(rows[0].find('a').text, '../') + + # Test proper escaping of files with funny names + self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt') + self.assertEqual(rows[1].find('a').text, 'foo::3.txt') + # Test files without escaping + self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt') + self.assertEqual(rows[2].find('a').text, 'subdir.txt') + + def test_topdir_parent_link(self): + '''Test index generation creates topdir parent link''' + with FileList() as fl: + fl.add(os.path.join(FIXTURE_DIR, 'logs/')) + ix = Indexer(fl) + ix.make_indexes( + create_parent_links=True, + create_topdir_parent_link=True) + + self.assert_files(fl, [ + ('', 'application/directory', None), + ('controller', 'application/directory', None), + ('zuul-info', 'application/directory', None), + ('job-output.json', 'application/json', None), + (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', + 'text/plain', None), + ('index.html', 'text/html', None), + ('controller/subdir', 'application/directory', None), + ('controller/compressed.gz', 'text/plain', 'gzip'), + ('controller/cpu-load.svg', 'image/svg+xml', None), + ('controller/journal.xz', 'text/plain', 'xz'), + ('controller/service_log.txt', 'text/plain', None), + ('controller/syslog', 'text/plain', None), + ('controller/index.html', 'text/html', None), + ('controller/subdir/foo::3.txt', 'text/plain', None), + ('controller/subdir/subdir.txt', 'text/plain', None), + ('controller/subdir/index.html', 'text/html', None), + ('zuul-info/inventory.yaml', 'text/plain', None), + ('zuul-info/zuul-info.controller.txt', 'text/plain', None), + ('zuul-info/index.html', 'text/html', None), + ]) + + top_index = self.find_file(fl, 'index.html') + page = open(top_index.full_path).read() + page = BeautifulSoup(page, 'html.parser') + rows = page.find_all('tr')[1:] + + self.assertEqual(len(rows), 5) + + self.assertEqual(rows[0].find('a').get('href'), + '../index.html') + self.assertEqual(rows[0].find('a').text, '../') + + self.assertEqual(rows[1].find('a').get('href'), + 'controller/index.html') + self.assertEqual(rows[1].find('a').text, 'controller/') + + self.assertEqual(rows[2].find('a').get('href'), + 'zuul-info/index.html') + self.assertEqual(rows[2].find('a').text, 'zuul-info/') + + subdir_index = self.find_file(fl, 'controller/subdir/index.html') + page = open(subdir_index.full_path).read() + page = BeautifulSoup(page, 'html.parser') + rows = page.find_all('tr')[1:] + self.assertEqual(rows[0].find('a').get('href'), '../index.html') + self.assertEqual(rows[0].find('a').text, '../') + + # Test proper escaping of files with funny names + self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt') + self.assertEqual(rows[1].find('a').text, 'foo::3.txt') + # Test files without escaping + self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt') + self.assertEqual(rows[2].find('a').text, 'subdir.txt') + + def test_no_parent_links(self): + '''Test index generation creates topdir parent link''' + with FileList() as fl: + fl.add(os.path.join(FIXTURE_DIR, 'logs/')) + ix = Indexer(fl) + ix.make_indexes( + create_parent_links=False, + create_topdir_parent_link=False) + + self.assert_files(fl, [ + ('', 'application/directory', None), + ('controller', 'application/directory', None), + ('zuul-info', 'application/directory', None), + ('job-output.json', 'application/json', None), + (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', + 'text/plain', None), + ('index.html', 'text/html', None), + ('controller/subdir', 'application/directory', None), + ('controller/compressed.gz', 'text/plain', 'gzip'), + ('controller/cpu-load.svg', 'image/svg+xml', None), + ('controller/journal.xz', 'text/plain', 'xz'), + ('controller/service_log.txt', 'text/plain', None), + ('controller/syslog', 'text/plain', None), + ('controller/index.html', 'text/html', None), + ('controller/subdir/foo::3.txt', 'text/plain', None), + ('controller/subdir/subdir.txt', 'text/plain', None), + ('controller/subdir/index.html', 'text/html', None), + ('zuul-info/inventory.yaml', 'text/plain', None), + ('zuul-info/zuul-info.controller.txt', 'text/plain', None), + ('zuul-info/index.html', 'text/html', None), + ]) + + top_index = self.find_file(fl, 'index.html') + page = open(top_index.full_path).read() + page = BeautifulSoup(page, 'html.parser') + rows = page.find_all('tr')[1:] + + self.assertEqual(len(rows), 4) + + self.assertEqual(rows[0].find('a').get('href'), + 'controller/index.html') + self.assertEqual(rows[0].find('a').text, + 'controller/') + + self.assertEqual(rows[1].find('a').get('href'), + 'zuul-info/index.html') + self.assertEqual(rows[1].find('a').text, + 'zuul-info/') + + subdir_index = self.find_file(fl, 'controller/subdir/index.html') + page = open(subdir_index.full_path).read() + page = BeautifulSoup(page, 'html.parser') + rows = page.find_all('tr')[1:] + + # Test proper escaping of files with funny names + self.assertEqual(rows[0].find('a').get('href'), 'foo%3A%3A3.txt') + self.assertEqual(rows[0].find('a').text, 'foo::3.txt') + # Test files without escaping + self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt') + self.assertEqual(rows[1].find('a').text, 'subdir.txt') + + +class TestFileDetail(testtools.TestCase): + + def test_get_file_detail(self): + '''Test files info''' + path = os.path.join(FIXTURE_DIR, 'logs/job-output.json') + file_detail = FileDetail(path, '') + path_stat = os.stat(path) + self.assertEqual( + time.gmtime(path_stat[stat.ST_MTIME]), + file_detail.last_modified) + self.assertEqual(16, file_detail.size) + + def test_get_file_detail_missing_file(self): + '''Test files that go missing during a walk''' + + file_detail = FileDetail('missing/file/that/we/cant/find', '') + + self.assertEqual(time.gmtime(0), file_detail.last_modified) + self.assertEqual(0, file_detail.size) + + +class MockUploader(Uploader): + """An uploader that uses a mocked cloud object""" + def __init__(self, container): + self.container = container + self.cloud = mock.Mock() + self.dry_run = False + self.public = True + self.delete_after = None + self.prefix = "" + self.url = 'http://dry-run-url.com/a/path/' + + +class TestUpload(testtools.TestCase): + + def test_upload_result(self): + uploader = MockUploader(container="container") + + # Let the upload method fail for the job-output.json, but not for the + # inventory.yaml file + def side_effect(container, name, **ignored): + if name == "job-output.json": + raise requests.exceptions.RequestException( + "Failed for a reason" + ) + uploader.cloud.create_object = mock.Mock( + side_effect=side_effect + ) + + # Get some test files to upload + files = [ + FileDetail( + os.path.join(FIXTURE_DIR, "logs/job-output.json"), + "job-output.json", + ), + FileDetail( + os.path.join(FIXTURE_DIR, "logs/zuul-info/inventory.yaml"), + "inventory.yaml", + ), + ] + + expected_failures = [ + { + "file": "job-output.json", + "error": ( + "Error posting file after multiple attempts: " + "Failed for a reason" + ), + }, + ] + + failures = uploader.upload(files) + self.assertEqual(expected_failures, failures) diff --git a/roles/upload-logs-base/library/test_zuul_google_storage_upload.py b/roles/upload-logs-base/library/test_zuul_google_storage_upload.py index 7a20e1c33..351e37203 100644 --- a/roles/upload-logs-base/library/test_zuul_google_storage_upload.py +++ b/roles/upload-logs-base/library/test_zuul_google_storage_upload.py @@ -19,16 +19,8 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type import os -import six import testtools -import time -import stat -import fixtures -from bs4 import BeautifulSoup -from testtools import skip - -from .zuul_google_storage_upload import FileList, Indexer, FileDetail from .zuul_google_storage_upload import Credentials @@ -36,396 +28,6 @@ FIXTURE_DIR = os.path.join(os.path.dirname(__file__), 'test-fixtures') -class SymlinkFixture(fixtures.Fixture): - links = [ - ('bad_symlink', '/etc'), - ('bad_symlink_file', '/etc/issue'), - ('good_symlink', 'controller'), - ('recursive_symlink', '.'), - ('symlink_file', 'job-output.json'), - ('symlink_loop_a', 'symlink_loop'), - ('symlink_loop/symlink_loop_b', '..'), - ] - - def _setUp(self): - for (src, target) in self.links: - path = os.path.join(FIXTURE_DIR, 'links', src) - os.symlink(target, path) - self.addCleanup(os.unlink, path) - - -class TestFileList(testtools.TestCase): - - def assert_files(self, result, files): - self.assertEqual(len(result), len(files)) - for expected, received in zip(files, result): - e = expected[0] - if six.PY2: - e = e.encode('utf-8') - self.assertEqual(e, received.relative_path) - if e and e[0][-1] == '/': - efilename = os.path.split( - os.path.dirname(e))[1] + '/' - else: - efilename = os.path.split(e)[1] - self.assertEqual(efilename, received.filename) - if received.folder: - if received.full_path is not None and expected[0] != '': - self.assertTrue(os.path.isdir(received.full_path)) - else: - self.assertTrue(os.path.isfile(received.full_path)) - self.assertEqual(expected[1], received.mimetype) - self.assertEqual(expected[2], received.encoding) - - def find_file(self, file_list, path): - for f in file_list: - if f.relative_path == path: - return f - - def test_single_dir_trailing_slash(self): - '''Test a single directory with a trailing slash''' - - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ]) - - def test_single_dir(self): - '''Test a single directory without a trailing slash''' - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs')) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('logs', 'application/directory', None), - ('logs/controller', 'application/directory', None), - ('logs/zuul-info', 'application/directory', None), - ('logs/job-output.json', 'application/json', None), - (u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('logs/controller/subdir', 'application/directory', None), - ('logs/controller/compressed.gz', 'text/plain', 'gzip'), - ('logs/controller/cpu-load.svg', 'image/svg+xml', None), - ('logs/controller/journal.xz', 'text/plain', 'xz'), - ('logs/controller/service_log.txt', 'text/plain', None), - ('logs/controller/syslog', 'text/plain', None), - ('logs/controller/subdir/foo::3.txt', 'text/plain', None), - ('logs/controller/subdir/subdir.txt', 'text/plain', None), - ('logs/zuul-info/inventory.yaml', 'text/plain', None), - ('logs/zuul-info/zuul-info.controller.txt', - 'text/plain', None), - ]) - - def test_single_file(self): - '''Test a single file''' - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, - 'logs/zuul-info/inventory.yaml')) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('inventory.yaml', 'text/plain', None), - ]) - - @skip('Temporarily disabled due to race conditions.') - def test_symlinks(self): - '''Test symlinks''' - with FileList() as fl: - self.useFixture(SymlinkFixture()) - fl.add(os.path.join(FIXTURE_DIR, 'links/')) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('good_symlink', 'application/directory', None), - ('recursive_symlink', 'application/directory', None), - ('symlink_loop', 'application/directory', None), - ('symlink_loop_a', 'application/directory', None), - ('job-output.json', 'application/json', None), - ('symlink_file', 'text/plain', None), - ('controller/service_log.txt', 'text/plain', None), - ('symlink_loop/symlink_loop_b', 'application/directory', None), - ('symlink_loop/placeholder', 'text/plain', None), - ]) - - def test_index_files(self): - '''Test index generation''' - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs')) - ix = Indexer(fl) - ix.make_indexes() - - self.assert_files(fl, [ - ('', 'application/directory', None), - ('index.html', 'text/html', None), - ('logs', 'application/directory', None), - ('logs/controller', 'application/directory', None), - ('logs/zuul-info', 'application/directory', None), - ('logs/job-output.json', 'application/json', None), - (u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('logs/index.html', 'text/html', None), - ('logs/controller/subdir', 'application/directory', None), - ('logs/controller/compressed.gz', 'text/plain', 'gzip'), - ('logs/controller/cpu-load.svg', 'image/svg+xml', None), - ('logs/controller/journal.xz', 'text/plain', 'xz'), - ('logs/controller/service_log.txt', 'text/plain', None), - ('logs/controller/syslog', 'text/plain', None), - ('logs/controller/index.html', 'text/html', None), - ('logs/controller/subdir/foo::3.txt', 'text/plain', None), - ('logs/controller/subdir/subdir.txt', 'text/plain', None), - ('logs/controller/subdir/index.html', 'text/html', None), - ('logs/zuul-info/inventory.yaml', 'text/plain', None), - ('logs/zuul-info/zuul-info.controller.txt', - 'text/plain', None), - ('logs/zuul-info/index.html', 'text/html', None), - ]) - - top_index = self.find_file(fl, 'index.html') - page = open(top_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - - self.assertEqual(len(rows), 1) - - self.assertEqual(rows[0].find('a').get('href'), 'logs/index.html') - self.assertEqual(rows[0].find('a').text, 'logs/') - - subdir_index = self.find_file( - fl, 'logs/controller/subdir/index.html') - page = open(subdir_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - self.assertEqual(rows[0].find('a').get('href'), '../index.html') - self.assertEqual(rows[0].find('a').text, '../') - - # Test proper escaping of files with funny names - self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt') - self.assertEqual(rows[1].find('a').text, 'foo::3.txt') - # Test files without escaping - self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt') - self.assertEqual(rows[2].find('a').text, 'subdir.txt') - - def test_index_files_trailing_slash(self): - '''Test index generation with a trailing slash''' - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - ix = Indexer(fl) - ix.make_indexes() - - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('index.html', 'text/html', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/index.html', 'text/html', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('controller/subdir/index.html', 'text/html', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ('zuul-info/index.html', 'text/html', None), - ]) - - top_index = self.find_file(fl, 'index.html') - page = open(top_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - - self.assertEqual(len(rows), 4) - - self.assertEqual(rows[0].find('a').get('href'), - 'controller/index.html') - self.assertEqual(rows[0].find('a').text, 'controller/') - - self.assertEqual(rows[1].find('a').get('href'), - 'zuul-info/index.html') - self.assertEqual(rows[1].find('a').text, 'zuul-info/') - - subdir_index = self.find_file(fl, 'controller/subdir/index.html') - page = open(subdir_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - self.assertEqual(rows[0].find('a').get('href'), '../index.html') - self.assertEqual(rows[0].find('a').text, '../') - - # Test proper escaping of files with funny names - self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt') - self.assertEqual(rows[1].find('a').text, 'foo::3.txt') - # Test files without escaping - self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt') - self.assertEqual(rows[2].find('a').text, 'subdir.txt') - - def test_topdir_parent_link(self): - '''Test index generation creates topdir parent link''' - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - ix = Indexer(fl) - ix.make_indexes( - create_parent_links=True, - create_topdir_parent_link=True) - - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('index.html', 'text/html', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/index.html', 'text/html', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('controller/subdir/index.html', 'text/html', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ('zuul-info/index.html', 'text/html', None), - ]) - - top_index = self.find_file(fl, 'index.html') - page = open(top_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - - self.assertEqual(len(rows), 5) - - self.assertEqual(rows[0].find('a').get('href'), - '../index.html') - self.assertEqual(rows[0].find('a').text, '../') - - self.assertEqual(rows[1].find('a').get('href'), - 'controller/index.html') - self.assertEqual(rows[1].find('a').text, 'controller/') - - self.assertEqual(rows[2].find('a').get('href'), - 'zuul-info/index.html') - self.assertEqual(rows[2].find('a').text, 'zuul-info/') - - subdir_index = self.find_file(fl, 'controller/subdir/index.html') - page = open(subdir_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - self.assertEqual(rows[0].find('a').get('href'), '../index.html') - self.assertEqual(rows[0].find('a').text, '../') - - # Test proper escaping of files with funny names - self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt') - self.assertEqual(rows[1].find('a').text, 'foo::3.txt') - # Test files without escaping - self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt') - self.assertEqual(rows[2].find('a').text, 'subdir.txt') - - def test_no_parent_links(self): - '''Test index generation creates topdir parent link''' - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - ix = Indexer(fl) - ix.make_indexes( - create_parent_links=False, - create_topdir_parent_link=False) - - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('index.html', 'text/html', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/index.html', 'text/html', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('controller/subdir/index.html', 'text/html', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ('zuul-info/index.html', 'text/html', None), - ]) - - top_index = self.find_file(fl, 'index.html') - page = open(top_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - - self.assertEqual(len(rows), 4) - - self.assertEqual(rows[0].find('a').get('href'), - 'controller/index.html') - self.assertEqual(rows[0].find('a').text, - 'controller/') - - self.assertEqual(rows[1].find('a').get('href'), - 'zuul-info/index.html') - self.assertEqual(rows[1].find('a').text, - 'zuul-info/') - - subdir_index = self.find_file(fl, 'controller/subdir/index.html') - page = open(subdir_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - - # Test proper escaping of files with funny names - self.assertEqual(rows[0].find('a').get('href'), 'foo%3A%3A3.txt') - self.assertEqual(rows[0].find('a').text, 'foo::3.txt') - # Test files without escaping - self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt') - self.assertEqual(rows[1].find('a').text, 'subdir.txt') - - -class TestFileDetail(testtools.TestCase): - - def test_get_file_detail(self): - '''Test files info''' - path = os.path.join(FIXTURE_DIR, 'logs/job-output.json') - file_detail = FileDetail(path, '') - path_stat = os.stat(path) - self.assertEqual( - time.gmtime(path_stat[stat.ST_MTIME]), - file_detail.last_modified) - self.assertEqual(16, file_detail.size) - - def test_get_file_detail_missing_file(self): - '''Test files that go missing during a walk''' - - file_detail = FileDetail('missing/file/that/we/cant/find', '') - - self.assertEqual(time.gmtime(0), file_detail.last_modified) - self.assertEqual(0, file_detail.size) - - class TestCredential(testtools.TestCase): def test_credential(self): diff --git a/roles/upload-logs-base/library/test_zuul_s3_upload.py b/roles/upload-logs-base/library/test_zuul_s3_upload.py index e208f0c75..b0eb64753 100644 --- a/roles/upload-logs-base/library/test_zuul_s3_upload.py +++ b/roles/upload-logs-base/library/test_zuul_s3_upload.py @@ -6,407 +6,11 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type import os -import six import testtools -import time -import stat -import fixtures - -from bs4 import BeautifulSoup -from testtools import skip - -from .zuul_s3_upload import FileList, Indexer, FileDetail - - FIXTURE_DIR = os.path.join(os.path.dirname(__file__), 'test-fixtures') -class SymlinkFixture(fixtures.Fixture): - links = [ - ('bad_symlink', '/etc'), - ('bad_symlink_file', '/etc/issue'), - ('good_symlink', 'controller'), - ('recursive_symlink', '.'), - ('symlink_file', 'job-output.json'), - ('symlink_loop_a', 'symlink_loop'), - ('symlink_loop/symlink_loop_b', '..'), - ] - - def _setUp(self): - for (src, target) in self.links: - path = os.path.join(FIXTURE_DIR, 'links', src) - os.symlink(target, path) - self.addCleanup(os.unlink, path) - - -class TestFileList(testtools.TestCase): - - def assert_files(self, result, files): - self.assertEqual(len(result), len(files)) - for expected, received in zip(files, result): - e = expected[0] - if six.PY2: - e = e.encode('utf-8') - self.assertEqual(e, received.relative_path) - if e and e[0][-1] == '/': - efilename = os.path.split( - os.path.dirname(e))[1] + '/' - else: - efilename = os.path.split(e)[1] - self.assertEqual(efilename, received.filename) - if received.folder: - if received.full_path is not None and expected[0] != '': - self.assertTrue(os.path.isdir(received.full_path)) - else: - self.assertTrue(os.path.isfile(received.full_path)) - self.assertEqual(expected[1], received.mimetype) - self.assertEqual(expected[2], received.encoding) - - def find_file(self, file_list, path): - for f in file_list: - if f.relative_path == path: - return f - - def test_single_dir_trailing_slash(self): - '''Test a single directory with a trailing slash''' - - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ]) - - def test_single_dir(self): - '''Test a single directory without a trailing slash''' - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs')) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('logs', 'application/directory', None), - ('logs/controller', 'application/directory', None), - ('logs/zuul-info', 'application/directory', None), - ('logs/job-output.json', 'application/json', None), - (u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('logs/controller/subdir', 'application/directory', None), - ('logs/controller/compressed.gz', 'text/plain', 'gzip'), - ('logs/controller/cpu-load.svg', 'image/svg+xml', None), - ('logs/controller/journal.xz', 'text/plain', 'xz'), - ('logs/controller/service_log.txt', 'text/plain', None), - ('logs/controller/syslog', 'text/plain', None), - ('logs/controller/subdir/foo::3.txt', 'text/plain', None), - ('logs/controller/subdir/subdir.txt', 'text/plain', None), - ('logs/zuul-info/inventory.yaml', 'text/plain', None), - ('logs/zuul-info/zuul-info.controller.txt', - 'text/plain', None), - ]) - - def test_single_file(self): - '''Test a single file''' - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, - 'logs/zuul-info/inventory.yaml')) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('inventory.yaml', 'text/plain', None), - ]) - - @skip('Temporarily disabled due to race conditions.') - def test_symlinks(self): - '''Test symlinks''' - with FileList() as fl: - self.useFixture(SymlinkFixture()) - fl.add(os.path.join(FIXTURE_DIR, 'links/')) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('good_symlink', 'application/directory', None), - ('recursive_symlink', 'application/directory', None), - ('symlink_loop', 'application/directory', None), - ('symlink_loop_a', 'application/directory', None), - ('job-output.json', 'application/json', None), - ('symlink_file', 'text/plain', None), - ('controller/service_log.txt', 'text/plain', None), - ('symlink_loop/symlink_loop_b', 'application/directory', None), - ('symlink_loop/placeholder', 'text/plain', None), - ]) - - def test_index_files(self): - '''Test index generation''' - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs')) - ix = Indexer(fl) - ix.make_indexes() - - self.assert_files(fl, [ - ('', 'application/directory', None), - ('index.html', 'text/html', None), - ('logs', 'application/directory', None), - ('logs/controller', 'application/directory', None), - ('logs/zuul-info', 'application/directory', None), - ('logs/job-output.json', 'application/json', None), - (u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('logs/index.html', 'text/html', None), - ('logs/controller/subdir', 'application/directory', None), - ('logs/controller/compressed.gz', 'text/plain', 'gzip'), - ('logs/controller/cpu-load.svg', 'image/svg+xml', None), - ('logs/controller/journal.xz', 'text/plain', 'xz'), - ('logs/controller/service_log.txt', 'text/plain', None), - ('logs/controller/syslog', 'text/plain', None), - ('logs/controller/index.html', 'text/html', None), - ('logs/controller/subdir/foo::3.txt', 'text/plain', None), - ('logs/controller/subdir/subdir.txt', 'text/plain', None), - ('logs/controller/subdir/index.html', 'text/html', None), - ('logs/zuul-info/inventory.yaml', 'text/plain', None), - ('logs/zuul-info/zuul-info.controller.txt', - 'text/plain', None), - ('logs/zuul-info/index.html', 'text/html', None), - ]) - - top_index = self.find_file(fl, 'index.html') - page = open(top_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - - self.assertEqual(len(rows), 1) - - self.assertEqual(rows[0].find('a').get('href'), 'logs/index.html') - self.assertEqual(rows[0].find('a').text, 'logs/') - - subdir_index = self.find_file( - fl, 'logs/controller/subdir/index.html') - page = open(subdir_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - self.assertEqual(rows[0].find('a').get('href'), '../index.html') - self.assertEqual(rows[0].find('a').text, '../') - - # Test proper escaping of files with funny names - self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt') - self.assertEqual(rows[1].find('a').text, 'foo::3.txt') - # Test files without escaping - self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt') - self.assertEqual(rows[2].find('a').text, 'subdir.txt') - - def test_index_files_trailing_slash(self): - '''Test index generation with a trailing slash''' - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - ix = Indexer(fl) - ix.make_indexes() - - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('index.html', 'text/html', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/index.html', 'text/html', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('controller/subdir/index.html', 'text/html', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ('zuul-info/index.html', 'text/html', None), - ]) - - top_index = self.find_file(fl, 'index.html') - page = open(top_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - - self.assertEqual(len(rows), 4) - - self.assertEqual(rows[0].find('a').get('href'), - 'controller/index.html') - self.assertEqual(rows[0].find('a').text, 'controller/') - - self.assertEqual(rows[1].find('a').get('href'), - 'zuul-info/index.html') - self.assertEqual(rows[1].find('a').text, 'zuul-info/') - - subdir_index = self.find_file(fl, 'controller/subdir/index.html') - page = open(subdir_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - self.assertEqual(rows[0].find('a').get('href'), '../index.html') - self.assertEqual(rows[0].find('a').text, '../') - - # Test proper escaping of files with funny names - self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt') - self.assertEqual(rows[1].find('a').text, 'foo::3.txt') - # Test files without escaping - self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt') - self.assertEqual(rows[2].find('a').text, 'subdir.txt') - - def test_topdir_parent_link(self): - '''Test index generation creates topdir parent link''' - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - ix = Indexer(fl) - ix.make_indexes( - create_parent_links=True, - create_topdir_parent_link=True) - - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('index.html', 'text/html', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/index.html', 'text/html', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('controller/subdir/index.html', 'text/html', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ('zuul-info/index.html', 'text/html', None), - ]) - - top_index = self.find_file(fl, 'index.html') - page = open(top_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - - self.assertEqual(len(rows), 5) - - self.assertEqual(rows[0].find('a').get('href'), - '../index.html') - self.assertEqual(rows[0].find('a').text, '../') - - self.assertEqual(rows[1].find('a').get('href'), - 'controller/index.html') - self.assertEqual(rows[1].find('a').text, 'controller/') - - self.assertEqual(rows[2].find('a').get('href'), - 'zuul-info/index.html') - self.assertEqual(rows[2].find('a').text, 'zuul-info/') - - subdir_index = self.find_file(fl, 'controller/subdir/index.html') - page = open(subdir_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - self.assertEqual(rows[0].find('a').get('href'), '../index.html') - self.assertEqual(rows[0].find('a').text, '../') - - # Test proper escaping of files with funny names - self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt') - self.assertEqual(rows[1].find('a').text, 'foo::3.txt') - # Test files without escaping - self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt') - self.assertEqual(rows[2].find('a').text, 'subdir.txt') - - def test_no_parent_links(self): - '''Test index generation creates topdir parent link''' - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - ix = Indexer(fl) - ix.make_indexes( - create_parent_links=False, - create_topdir_parent_link=False) - - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('index.html', 'text/html', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/index.html', 'text/html', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('controller/subdir/index.html', 'text/html', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ('zuul-info/index.html', 'text/html', None), - ]) - - top_index = self.find_file(fl, 'index.html') - page = open(top_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - - self.assertEqual(len(rows), 4) - - self.assertEqual(rows[0].find('a').get('href'), - 'controller/index.html') - self.assertEqual(rows[0].find('a').text, - 'controller/') - - self.assertEqual(rows[1].find('a').get('href'), - 'zuul-info/index.html') - self.assertEqual(rows[1].find('a').text, - 'zuul-info/') - - subdir_index = self.find_file(fl, 'controller/subdir/index.html') - page = open(subdir_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - - # Test proper escaping of files with funny names - self.assertEqual(rows[0].find('a').get('href'), 'foo%3A%3A3.txt') - self.assertEqual(rows[0].find('a').text, 'foo::3.txt') - # Test files without escaping - self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt') - self.assertEqual(rows[1].find('a').text, 'subdir.txt') - - -class TestFileDetail(testtools.TestCase): - - def test_get_file_detail(self): - '''Test files info''' - path = os.path.join(FIXTURE_DIR, 'logs/job-output.json') - file_detail = FileDetail(path, '') - path_stat = os.stat(path) - self.assertEqual( - time.gmtime(path_stat[stat.ST_MTIME]), - file_detail.last_modified) - self.assertEqual(16, file_detail.size) - - def test_get_file_detail_missing_file(self): - '''Test files that go missing during a walk''' - - file_detail = FileDetail('missing/file/that/we/cant/find', '') - - self.assertEqual(time.gmtime(0), file_detail.last_modified) - self.assertEqual(0, file_detail.size) +class TestUpload(testtools.TestCase): + # TODO: Implement upload test case + pass diff --git a/roles/upload-logs-base/library/test_zuul_swift_upload.py b/roles/upload-logs-base/library/test_zuul_swift_upload.py index cf088987a..712bb042f 100644 --- a/roles/upload-logs-base/library/test_zuul_swift_upload.py +++ b/roles/upload-logs-base/library/test_zuul_swift_upload.py @@ -19,405 +19,20 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type import os -import six import testtools -import time -import stat -import fixtures try: from unittest import mock except ImportError: import mock import requests -from bs4 import BeautifulSoup -from .zuul_swift_upload import FileList, Indexer, FileDetail, Uploader - +from .zuul_swift_upload import Uploader +from ..module_utils.zuul_jobs.upload_utils import FileDetail FIXTURE_DIR = os.path.join(os.path.dirname(__file__), 'test-fixtures') -class SymlinkFixture(fixtures.Fixture): - links = [ - ('bad_symlink', '/etc'), - ('bad_symlink_file', '/etc/issue'), - ('good_symlink', 'controller'), - ('recursive_symlink', '.'), - ('symlink_file', 'job-output.json'), - ('symlink_loop_a', 'symlink_loop'), - ('symlink_loop/symlink_loop_b', '..'), - ] - - def _setUp(self): - for (src, target) in self.links: - path = os.path.join(FIXTURE_DIR, 'links', src) - os.symlink(target, path) - self.addCleanup(os.unlink, path) - - -class TestFileList(testtools.TestCase): - - def assert_files(self, result, files): - self.assertEqual(len(result), len(files)) - for expected, received in zip(files, result): - e = expected[0] - if six.PY2: - e = e.encode('utf-8') - self.assertEqual(e, received.relative_path) - if e and e[0][-1] == '/': - efilename = os.path.split( - os.path.dirname(e))[1] + '/' - else: - efilename = os.path.split(e)[1] - self.assertEqual(efilename, received.filename) - if received.folder: - if received.full_path is not None and expected[0] != '': - self.assertTrue(os.path.isdir(received.full_path)) - else: - self.assertTrue(os.path.isfile(received.full_path)) - self.assertEqual(expected[1], received.mimetype) - self.assertEqual(expected[2], received.encoding) - - def find_file(self, file_list, path): - for f in file_list: - if f.relative_path == path: - return f - - def test_single_dir_trailing_slash(self): - '''Test a single directory with a trailing slash''' - - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ]) - - def test_single_dir(self): - '''Test a single directory without a trailing slash''' - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs')) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('logs', 'application/directory', None), - ('logs/controller', 'application/directory', None), - ('logs/zuul-info', 'application/directory', None), - ('logs/job-output.json', 'application/json', None), - (u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('logs/controller/subdir', 'application/directory', None), - ('logs/controller/compressed.gz', 'text/plain', 'gzip'), - ('logs/controller/cpu-load.svg', 'image/svg+xml', None), - ('logs/controller/journal.xz', 'text/plain', 'xz'), - ('logs/controller/service_log.txt', 'text/plain', None), - ('logs/controller/syslog', 'text/plain', None), - ('logs/controller/subdir/foo::3.txt', 'text/plain', None), - ('logs/controller/subdir/subdir.txt', 'text/plain', None), - ('logs/zuul-info/inventory.yaml', 'text/plain', None), - ('logs/zuul-info/zuul-info.controller.txt', - 'text/plain', None), - ]) - - def test_single_file(self): - '''Test a single file''' - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, - 'logs/zuul-info/inventory.yaml')) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('inventory.yaml', 'text/plain', None), - ]) - - def test_symlinks(self): - '''Test symlinks''' - with FileList() as fl: - self.useFixture(SymlinkFixture()) - fl.add(os.path.join(FIXTURE_DIR, 'links/')) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('good_symlink', 'application/directory', None), - ('recursive_symlink', 'application/directory', None), - ('symlink_loop', 'application/directory', None), - ('symlink_loop_a', 'application/directory', None), - ('job-output.json', 'application/json', None), - ('symlink_file', 'text/plain', None), - ('controller/service_log.txt', 'text/plain', None), - ('symlink_loop/symlink_loop_b', 'application/directory', None), - ('symlink_loop/placeholder', 'text/plain', None), - ]) - - def test_index_files(self): - '''Test index generation''' - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs')) - ix = Indexer(fl) - ix.make_indexes() - - self.assert_files(fl, [ - ('', 'application/directory', None), - ('index.html', 'text/html', None), - ('logs', 'application/directory', None), - ('logs/controller', 'application/directory', None), - ('logs/zuul-info', 'application/directory', None), - ('logs/job-output.json', 'application/json', None), - (u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('logs/index.html', 'text/html', None), - ('logs/controller/subdir', 'application/directory', None), - ('logs/controller/compressed.gz', 'text/plain', 'gzip'), - ('logs/controller/cpu-load.svg', 'image/svg+xml', None), - ('logs/controller/journal.xz', 'text/plain', 'xz'), - ('logs/controller/service_log.txt', 'text/plain', None), - ('logs/controller/syslog', 'text/plain', None), - ('logs/controller/index.html', 'text/html', None), - ('logs/controller/subdir/foo::3.txt', 'text/plain', None), - ('logs/controller/subdir/subdir.txt', 'text/plain', None), - ('logs/controller/subdir/index.html', 'text/html', None), - ('logs/zuul-info/inventory.yaml', 'text/plain', None), - ('logs/zuul-info/zuul-info.controller.txt', - 'text/plain', None), - ('logs/zuul-info/index.html', 'text/html', None), - ]) - - top_index = self.find_file(fl, 'index.html') - page = open(top_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - - self.assertEqual(len(rows), 1) - - self.assertEqual(rows[0].find('a').get('href'), 'logs/') - self.assertEqual(rows[0].find('a').text, 'logs/') - - subdir_index = self.find_file( - fl, 'logs/controller/subdir/index.html') - page = open(subdir_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - self.assertEqual(rows[0].find('a').get('href'), '../') - self.assertEqual(rows[0].find('a').text, '../') - - # Test proper escaping of files with funny names - self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt') - self.assertEqual(rows[1].find('a').text, 'foo::3.txt') - # Test files without escaping - self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt') - self.assertEqual(rows[2].find('a').text, 'subdir.txt') - - def test_index_files_trailing_slash(self): - '''Test index generation with a trailing slash''' - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - ix = Indexer(fl) - ix.make_indexes() - - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('index.html', 'text/html', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/index.html', 'text/html', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('controller/subdir/index.html', 'text/html', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ('zuul-info/index.html', 'text/html', None), - ]) - - top_index = self.find_file(fl, 'index.html') - page = open(top_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - - self.assertEqual(len(rows), 4) - - self.assertEqual(rows[0].find('a').get('href'), 'controller/') - self.assertEqual(rows[0].find('a').text, 'controller/') - - self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/') - self.assertEqual(rows[1].find('a').text, 'zuul-info/') - - subdir_index = self.find_file(fl, 'controller/subdir/index.html') - page = open(subdir_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - self.assertEqual(rows[0].find('a').get('href'), '../') - self.assertEqual(rows[0].find('a').text, '../') - - # Test proper escaping of files with funny names - self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt') - self.assertEqual(rows[1].find('a').text, 'foo::3.txt') - # Test files without escaping - self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt') - self.assertEqual(rows[2].find('a').text, 'subdir.txt') - - def test_topdir_parent_link(self): - '''Test index generation creates topdir parent link''' - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - ix = Indexer(fl) - ix.make_indexes( - create_parent_links=True, - create_topdir_parent_link=True) - - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('index.html', 'text/html', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/index.html', 'text/html', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('controller/subdir/index.html', 'text/html', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ('zuul-info/index.html', 'text/html', None), - ]) - - top_index = self.find_file(fl, 'index.html') - page = open(top_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - - self.assertEqual(len(rows), 5) - - self.assertEqual(rows[0].find('a').get('href'), '../') - self.assertEqual(rows[0].find('a').text, '../') - - self.assertEqual(rows[1].find('a').get('href'), 'controller/') - self.assertEqual(rows[1].find('a').text, 'controller/') - - self.assertEqual(rows[2].find('a').get('href'), 'zuul-info/') - self.assertEqual(rows[2].find('a').text, 'zuul-info/') - - subdir_index = self.find_file(fl, 'controller/subdir/index.html') - page = open(subdir_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - self.assertEqual(rows[0].find('a').get('href'), '../') - self.assertEqual(rows[0].find('a').text, '../') - - # Test proper escaping of files with funny names - self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt') - self.assertEqual(rows[1].find('a').text, 'foo::3.txt') - # Test files without escaping - self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt') - self.assertEqual(rows[2].find('a').text, 'subdir.txt') - - def test_no_parent_links(self): - '''Test index generation creates topdir parent link''' - with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - ix = Indexer(fl) - ix.make_indexes( - create_parent_links=False, - create_topdir_parent_link=False) - - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('index.html', 'text/html', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/index.html', 'text/html', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('controller/subdir/index.html', 'text/html', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ('zuul-info/index.html', 'text/html', None), - ]) - - top_index = self.find_file(fl, 'index.html') - page = open(top_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - - self.assertEqual(len(rows), 4) - - self.assertEqual(rows[0].find('a').get('href'), 'controller/') - self.assertEqual(rows[0].find('a').text, 'controller/') - - self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/') - self.assertEqual(rows[1].find('a').text, 'zuul-info/') - - subdir_index = self.find_file(fl, 'controller/subdir/index.html') - page = open(subdir_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - - # Test proper escaping of files with funny names - self.assertEqual(rows[0].find('a').get('href'), 'foo%3A%3A3.txt') - self.assertEqual(rows[0].find('a').text, 'foo::3.txt') - # Test files without escaping - self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt') - self.assertEqual(rows[1].find('a').text, 'subdir.txt') - - -class TestFileDetail(testtools.TestCase): - - def test_get_file_detail(self): - '''Test files info''' - path = os.path.join(FIXTURE_DIR, 'logs/job-output.json') - file_detail = FileDetail(path, '') - path_stat = os.stat(path) - self.assertEqual( - time.gmtime(path_stat[stat.ST_MTIME]), - file_detail.last_modified) - self.assertEqual(16, file_detail.size) - - def test_get_file_detail_missing_file(self): - '''Test files that go missing during a walk''' - - file_detail = FileDetail('missing/file/that/we/cant/find', '') - - self.assertEqual(time.gmtime(0), file_detail.last_modified) - self.assertEqual(0, file_detail.size) - - class MockUploader(Uploader): """An uploader that uses a mocked cloud object""" def __init__(self, container): diff --git a/roles/upload-logs-base/library/zuul_google_storage_upload.py b/roles/upload-logs-base/library/zuul_google_storage_upload.py index 88ccd2eea..7d80f27ce 100755 --- a/roles/upload-logs-base/library/zuul_google_storage_upload.py +++ b/roles/upload-logs-base/library/zuul_google_storage_upload.py @@ -26,28 +26,15 @@ Utility to upload files to google import argparse import datetime -import gzip -import io import json import logging -import mimetypes import os try: import queue as queuelib except ImportError: import Queue as queuelib -import shutil -import stat import sys -import tempfile import threading -import time -try: - import urllib.parse as urlparse -except ImportError: - import urllib as urlparse -import zlib -import collections from google.cloud import storage import google.auth.compute_engine.credentials as gce_cred @@ -55,207 +42,23 @@ import google.auth.compute_engine.credentials as gce_cred from ansible.module_utils.basic import AnsibleModule try: - # Python 3.3+ - from collections.abc import Sequence + # Ansible context + from ansible.module_utils.zuul_jobs.upload_utils import ( + FileList, + GZIPCompressedStream, + Indexer, + retry_function, + ) except ImportError: - from collections import Sequence - -mimetypes.init() -mimetypes.add_type('text/plain', '.yaml') + # Test context + from ..module_utils.zuul_jobs.upload_utils import ( + FileList, + GZIPCompressedStream, + Indexer, + retry_function, + ) MAX_UPLOAD_THREADS = 24 -POST_ATTEMPTS = 3 - -# Map mime types to apache icons -APACHE_MIME_ICON_MAP = { - '_default': 'unknown.png', - 'application/gzip': 'compressed.png', - 'application/directory': 'folder.png', - 'text/html': 'text.png', - 'text/plain': 'text.png', -} - -# Map mime types to apache icons -APACHE_FILE_ICON_MAP = { - '..': 'back.png', -} - -# These icon files are from the Apache project and are in the public -# domain. -ICON_IMAGES = { - 'back.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAElBMVEX/' - '///M//+ZmZlmZmYzMzMAAACei5rnAAAAAnRSTlP/AOW3MEoAAABWSURB' - 'VHjabdBBCgAhDEPRRpv7X3kwEMsQ//IRRC08urjRHbha5VLFUsVSxVI9' - 'lmDh5hMpHD6n0EgoiZG0DNINpnWlcVXaRix76e1/8dddcL6nG0Ri9gHj' - 'tgSXKYeLBgAAAABJRU5ErkJggg==', - 'compressed.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAADAFBM' - 'VEX//////8z//5n//2b//zP//wD/zP//zMz/zJn/zGb/zDP/zAD/' - 'mf//mcz/mZn/mWb/mTP/mQD/Zv//Zsz/Zpn/Zmb/ZjP/ZgD/M///' - 'M8z/M5n/M2b/MzP/MwD/AP//AMz/AJn/AGb/ADP/AADM///M/8zM' - '/5nM/2bM/zPM/wDMzP/MzMzMzJnMzGbMzDPMzADMmf/MmczMmZnM' - 'mWbMmTPMmQDMZv/MZszMZpnMZmbMZjPMZgDMM//MM8zMM5nMM2bM' - 'MzPMMwDMAP/MAMzMAJnMAGbMADPMAACZ//+Z/8yZ/5mZ/2aZ/zOZ' - '/wCZzP+ZzMyZzJmZzGaZzDOZzACZmf+ZmcyZmZmZmWaZmTOZmQCZ' - 'Zv+ZZsyZZpmZZmaZZjOZZgCZM/+ZM8yZM5mZM2aZMzOZMwCZAP+Z' - 'AMyZAJmZAGaZADOZAABm//9m/8xm/5lm/2Zm/zNm/wBmzP9mzMxm' - 'zJlmzGZmzDNmzABmmf9mmcxmmZlmmWZmmTNmmQBmZv9mZsxmZplm' - 'ZmZmZjNmZgBmM/9mM8xmM5lmM2ZmMzNmMwBmAP9mAMxmAJlmAGZm' - 'ADNmAAAz//8z/8wz/5kz/2Yz/zMz/wAzzP8zzMwzzJkzzGYzzDMz' - 'zAAzmf8zmcwzmZkzmWYzmTMzmQAzZv8zZswzZpkzZmYzZjMzZgAz' - 'M/8zM8wzM5kzM2YzMzMzMwAzAP8zAMwzAJkzAGYzADMzAAAA//8A' - '/8wA/5kA/2YA/zMA/wAAzP8AzMwAzJkAzGYAzDMAzAAAmf8AmcwA' - 'mZkAmWYAmTMAmQAAZv8AZswAZpkAZmYAZjMAZgAAM/8AM8wAM5kA' - 'M2YAMzMAMwAAAP8AAMwAAJkAAGYAADPuAADdAAC7AACqAACIAAB3' - 'AABVAABEAAAiAAARAAAA7gAA3QAAuwAAqgAAiAAAdwAAVQAARAAA' - 'IgAAEQAAAO4AAN0AALsAAKoAAIgAAHcAAFUAAEQAACIAABHu7u7d' - '3d27u7uqqqqIiIh3d3dVVVVEREQiIiIREREAAAD7CIKZAAAAJXRS' - 'TlP///////////////////////////////////////////////8A' - 'P89CTwAAAGtJREFUeNp9z9ENgDAIhOEOco+dybVuEXasFMRDY/x5' - '+xJCO6Znu6kSx7BhXyjtKBWWNlwW88Loid7hFRKBXiIYCMfMEYUQ' - 'QohC3CjFA5nIjqx1CqlDLGR/EhM5O06yvin0ftGOyIS7lV14AsQN' - 'aR7rMEBYAAAAAElFTkSuQmCC', - 'folder.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAElBMVEX/' - '////zJnM//+ZZjMzMzMAAADCEvqoAAAAA3RSTlP//wDXyg1BAAAASElE' - 'QVR42s3KQQ6AQAhDUaXt/a/sQDrRJu7c+NmQB0e99B3lnqjT6cYx6zSI' - 'bV40n3D7psYMoBoz4w8/EdNYQsbGEjNxYSljXTEsA9O1pLTvAAAAAElF' - 'TkSuQmCC', - 'text.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAD1BMVEX/' - '///M//+ZmZkzMzMAAABVsTOVAAAAAnRSTlP/AOW3MEoAAABISURBVHja' - 'tcrRCgAgCENRbf7/N7dKomGvngjhMsPLD4NdMPwia438NRIyxsaL/XQZ' - 'hyxpkC6zyjLXGVXnkhqWJWIIrOgeinECLlUCjBCqNQoAAAAASUVORK5C' - 'YII=', - 'unknown.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAD1BMVEX/' - '///M//+ZmZkzMzMAAABVsTOVAAAAAnRSTlP/AOW3MEoAAABYSURBVHja' - 'ncvRDoAgDEPRruX/v1kmNHPBxMTLyzgD6FmsILg56g2hQnJkOco4yZhq' - 'tN5nYd5Zq0LsHblwxwP9GTCWsaGtoelANKzOlz/RfaLYUmLE6E28ALlN' - 'AupSdoFsAAAAAElFTkSuQmCC'} - - -# Begin vendored code -# This code is licensed under the Public Domain/CC0 and comes from -# https://github.com/leenr/gzip-stream/blob/master/gzip_stream.py -# Code was modified: -# removed type annotations to support python2. -# removed use of *, somearg for positional anonymous args. -# Default compression level to 9. - -class GZIPCompressedStream(io.RawIOBase): - def __init__(self, stream, compression_level=9): - assert 1 <= compression_level <= 9 - - self._compression_level = compression_level - self._stream = stream - - self._compressed_stream = io.BytesIO() - self._compressor = gzip.GzipFile( - mode='wb', - fileobj=self._compressed_stream, - compresslevel=compression_level - ) - - # because of the GZIP header written by `GzipFile.__init__`: - self._compressed_stream.seek(0) - self.count = 0 - - def read(self, length): - r = super().read(length) - self.count += len(r) - return r - - def tell(self): - return self.count - - @property - def compression_level(self): - return self._compression_level - - @property - def stream(self): - return self._stream - - def readable(self): - return True - - def _read_compressed_into(self, b): - buf = self._compressed_stream.read(len(b)) - b[:len(buf)] = buf - return len(buf) - - def readinto(self, b): - b = memoryview(b) - - offset = 0 - size = len(b) - while offset < size: - offset += self._read_compressed_into(b[offset:]) - if offset < size: - # self._compressed_buffer now empty - if self._compressor.closed: - # nothing to compress anymore - break - # compress next bytes - self._read_n_compress(size) - - return offset - - def _read_n_compress(self, size): - assert size > 0 - - data = self._stream.read(size) - - # rewind buffer to the start to free up memory - # (because anything currently in the buffer should be already - # streamed off the object) - self._compressed_stream.seek(0) - self._compressed_stream.truncate(0) - - if data: - self._compressor.write(data) - else: - # this will write final data (will flush zlib with Z_FINISH) - self._compressor.close() - - # rewind to the buffer start - self._compressed_stream.seek(0) - - def __repr__(self): - return ( - '{self.__class__.__name__}(' - '{self.stream!r}, ' - 'compression_level={self.compression_level!r}' - ')' - ).format(self=self) - -# End vendored code - - -def get_mime_icon(mime, filename=''): - icon = (APACHE_FILE_ICON_MAP.get(filename) or - APACHE_MIME_ICON_MAP.get(mime) or - APACHE_MIME_ICON_MAP['_default']) - return "data:image/png;base64,%s" % ICON_IMAGES[icon] - - -def retry_function(func): - for attempt in range(1, POST_ATTEMPTS + 1): - try: - return func() - except Exception: - if attempt >= POST_ATTEMPTS: - raise - else: - logging.exception("Error on attempt %d" % attempt) - time.sleep(attempt * 10) - - -def sizeof_fmt(num, suffix='B'): - # From http://stackoverflow.com/questions/1094841/ - # reusable-library-to-get-human-readable-version-of-file-size - for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: - if abs(num) < 1024.0: - return "%3.1f%s%s" % (num, unit, suffix) - num /= 1024.0 - return "%.1f%s%s" % (num, 'Y', suffix) class Credentials(gce_cred.Credentials): @@ -272,351 +75,6 @@ class Credentials(gce_cred.Credentials): datetime.timedelta(seconds=data['expires_in'])) -class FileDetail(): - """ - Used to generate indexes with links or as the file path - to push to storage. - """ - - def __init__(self, full_path, relative_path, filename=None): - """ - Args: - full_path (str): The absolute path to the file on disk. - relative_path (str): The relative path from the artifacts source - used for links. - filename (str): An optional alternate filename in links. - """ - # Make FileNotFoundError exception to be compatible with python2 - try: - FileNotFoundError # noqa: F823 - except NameError: - FileNotFoundError = OSError - - self.full_path = full_path - if filename is None: - self.filename = os.path.basename(full_path) - else: - self.filename = filename - self.relative_path = relative_path - - if self.full_path and os.path.isfile(self.full_path): - mime_guess, encoding = mimetypes.guess_type(self.full_path) - self.mimetype = mime_guess if mime_guess else 'text/plain' - self.encoding = encoding - self.folder = False - else: - self.mimetype = 'application/directory' - self.encoding = None - self.folder = True - try: - st = os.stat(self.full_path) - self.last_modified = time.gmtime(st[stat.ST_MTIME]) - self.size = st[stat.ST_SIZE] - except (FileNotFoundError, TypeError): - self.last_modified = time.gmtime(0) - self.size = 0 - - def __repr__(self): - t = 'Folder' if self.folder else 'File' - return '<%s %s>' % (t, self.relative_path) - - -class FileList(Sequence): - '''A collection of FileDetail objects - - This is a list-like group of FileDetail objects, intended to be - used as a context manager around the upload process. - ''' - def __init__(self): - self.file_list = [] - self.file_list.append(FileDetail(None, '', '')) - self.tempdirs = [] - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - for tempdir in self.tempdirs: - shutil.rmtree(tempdir) - - def __getitem__(self, item): - return self.file_list.__getitem__(item) - - def __len__(self): - return self.file_list.__len__() - - def get_tempdir(self): - '''Get a temporary directory - - Returns path to a private temporary directory which will be - cleaned on exit - ''' - tempdir = tempfile.mkdtemp(prefix='s-u-l-tmp') - self.tempdirs.append(tempdir) - return tempdir - - @staticmethod - def _path_in_tree(root, path): - full_path = os.path.realpath(os.path.abspath( - os.path.expanduser(path))) - if not full_path.startswith(root): - logging.debug("Skipping path outside root: %s" % (path,)) - return False - return True - - def add(self, file_path): - """ - Generate a list of files to upload to storage. Recurses through - directories - """ - - # file_list: A list of FileDetails to push to storage - file_list = [] - - if os.path.isfile(file_path): - relative_path = os.path.basename(file_path) - file_list.append(FileDetail(file_path, relative_path)) - elif os.path.isdir(file_path): - original_root = os.path.realpath(os.path.abspath( - os.path.expanduser(file_path))) - - parent_dir = os.path.dirname(file_path) - if not file_path.endswith('/'): - filename = os.path.basename(file_path) - full_path = file_path - relative_name = os.path.relpath(full_path, parent_dir) - file_list.append(FileDetail(full_path, relative_name, - filename)) - # TODO: this will copy the result of symlinked files, but - # it won't follow directory symlinks. If we add that, we - # should ensure that we don't loop. - for path, folders, files in os.walk(file_path): - # Sort folder in-place so that we recurse in order. - files.sort(key=lambda x: x.lower()) - folders.sort(key=lambda x: x.lower()) - # relative_path: The path between the given directory - # and the one being currently walked. - relative_path = os.path.relpath(path, parent_dir) - - for filename in folders: - full_path = os.path.join(path, filename) - if not self._path_in_tree(original_root, full_path): - continue - relative_name = os.path.relpath(full_path, parent_dir) - file_list.append(FileDetail(full_path, relative_name, - filename)) - - for filename in files: - full_path = os.path.join(path, filename) - if not self._path_in_tree(original_root, full_path): - continue - relative_name = os.path.relpath(full_path, parent_dir) - file_detail = FileDetail(full_path, relative_name) - file_list.append(file_detail) - - self.file_list += file_list - - -class Indexer(): - """Index a FileList - - Functions to generate indexes and other collated data for a - FileList - - - make_indexes() : make index.html in folders - """ - def __init__(self, file_list): - ''' - Args: - file_list (FileList): A FileList object with all files - to be indexed. - ''' - assert isinstance(file_list, FileList) - self.file_list = file_list - - def _make_index_file(self, folder_links, title, tempdir, append_footer): - """Writes an index into a file for pushing""" - for file_details in folder_links: - # Do not generate an index file if one exists already. - # This may be the case when uploading other machine generated - # content like python coverage info. - if self.index_filename == file_details.filename: - return - index_content = self._generate_log_index( - folder_links, title, append_footer) - fd = open(os.path.join(tempdir, self.index_filename), 'w') - fd.write(index_content) - return os.path.join(tempdir, self.index_filename) - - def _generate_log_index(self, folder_links, title, append_footer): - """Create an index of logfiles and links to them""" - - output = '%s\n' % title - output += '

%s

\n' % title - output += '' - output += '' - - file_details_to_append = None - for file_details in folder_links: - output += '' - output += ( - '' % ({ - 'm': file_details.mimetype, - 'i': get_mime_icon(file_details.mimetype, - file_details.filename), - })) - filename = file_details.filename - link_filename = filename - if file_details.folder: - filename += '/' - link_filename += '/index.html' - output += '' % ( - urlparse.quote(link_filename), - filename) - output += '' % time.asctime( - file_details.last_modified) - size = sizeof_fmt(file_details.size, suffix='') - output += '' % size - output += '\n' - - if (append_footer and - append_footer in file_details.filename): - file_details_to_append = file_details - - output += '
NameLast ModifiedSize
[ ]%s%s%s
' - - if file_details_to_append: - output += '
' - try: - with open(file_details_to_append.full_path, 'r') as f: - output += f.read() - except IOError: - logging.exception("Error opening file for appending") - - output += '\n' - return output - - def make_indexes(self, create_parent_links=True, - create_topdir_parent_link=False, - append_footer='index_footer.html'): - '''Make index.html files - - Iterate the file list and crete index.html files for folders - - Args: - create_parent_links (bool): Create parent links - create_topdir_parent_link (bool): Create topdir parent link - append_footer (str): Filename of a footer to append to each - generated page - - Return: - No value, the self.file_list will be updated - ''' - self.index_filename = 'index.html' - - folders = collections.OrderedDict() - for f in self.file_list: - if f.folder: - folders[f.relative_path] = [] - folder = os.path.dirname(os.path.dirname( - f.relative_path + '/')) - if folder == '/': - folder = '' - else: - folder = os.path.dirname(f.relative_path) - folders[folder].append(f) - - indexes = {} - parent_file_detail = FileDetail(None, '..', '..') - for folder, files in folders.items(): - # Don't add the pseudo-top-directory - if files and files[0].full_path is None: - files = files[1:] - if create_topdir_parent_link: - files = [parent_file_detail] + files - elif create_parent_links: - files = [parent_file_detail] + files - - # Do generate a link to the parent directory - full_path = self._make_index_file(files, 'Index of %s' % (folder,), - self.file_list.get_tempdir(), - append_footer) - - if full_path: - filename = os.path.basename(full_path) - relative_name = os.path.join(folder, filename) - indexes[folder] = FileDetail(full_path, relative_name) - - # This appends the index file at the end of the group of files - # for each directory. - new_list = [] - last_dirname = None - for f in reversed(list(self.file_list)): - if f.folder: - relative_path = f.relative_path + '/' - else: - relative_path = f.relative_path - dirname = os.path.dirname(relative_path) - if dirname == '/': - dirname = '' - if dirname != last_dirname: - index = indexes.pop(dirname, None) - if index: - new_list.append(index) - last_dirname = dirname - new_list.append(f) - new_list.reverse() - self.file_list.file_list = new_list - - -class GzipFilter(): - chunk_size = 16384 - - def __init__(self, infile): - self.gzipfile = GZIPCompressedStream(infile) - self.done = False - - def __iter__(self): - return self - - def __next__(self): - if self.done: - self.gzipfile.close() - raise StopIteration() - data = self.gzipfile.read(self.chunk_size) - if not data: - self.done = True - return data - - -class DeflateFilter(): - chunk_size = 16384 - - def __init__(self, infile): - self.infile = infile - self.encoder = zlib.compressobj() - self.done = False - - def __iter__(self): - return self - - def __next__(self): - if self.done: - raise StopIteration() - ret = b'' - while True: - data = self.infile.read(self.chunk_size) - if data: - ret = self.encoder.compress(data) - if ret: - break - else: - self.done = True - ret = self.encoder.flush() - break - return ret - - class Uploader(): def __init__(self, client, container, prefix=None, dry_run=False): diff --git a/roles/upload-logs-base/library/zuul_s3_upload.py b/roles/upload-logs-base/library/zuul_s3_upload.py index d6e9a9842..9f1666ae9 100755 --- a/roles/upload-logs-base/library/zuul_s3_upload.py +++ b/roles/upload-logs-base/library/zuul_s3_upload.py @@ -25,570 +25,36 @@ Utility to upload files to s3 """ import argparse -import gzip -import io import logging -import mimetypes import os try: import queue as queuelib except ImportError: import Queue as queuelib -import shutil -import stat import sys -import tempfile import threading -import time -try: - import urllib.parse as urlparse -except ImportError: - import urllib as urlparse -import zlib -import collections import boto3 from ansible.module_utils.basic import AnsibleModule try: - # Python 3.3+ - from collections.abc import Sequence + # Ansible context + from ansible.module_utils.zuul_jobs.upload_utils import ( + FileList, + GZIPCompressedStream, + Indexer, + retry_function, + ) except ImportError: - from collections import Sequence - - -mimetypes.init() -mimetypes.add_type('text/plain', '.yaml') + # Test context + from ..module_utils.zuul_jobs.upload_utils import ( + FileList, + GZIPCompressedStream, + Indexer, + retry_function, + ) MAX_UPLOAD_THREADS = 24 -POST_ATTEMPTS = 3 - -# Map mime types to apache icons -APACHE_MIME_ICON_MAP = { - '_default': 'unknown.png', - 'application/gzip': 'compressed.png', - 'application/directory': 'folder.png', - 'text/html': 'text.png', - 'text/plain': 'text.png', -} - -# Map mime types to apache icons -APACHE_FILE_ICON_MAP = { - '..': 'back.png', -} - -# These icon files are from the Apache project and are in the public -# domain. -ICON_IMAGES = { - 'back.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAElBMVEX/' - '///M//+ZmZlmZmYzMzMAAACei5rnAAAAAnRSTlP/AOW3MEoAAABWSURB' - 'VHjabdBBCgAhDEPRRpv7X3kwEMsQ//IRRC08urjRHbha5VLFUsVSxVI9' - 'lmDh5hMpHD6n0EgoiZG0DNINpnWlcVXaRix76e1/8dddcL6nG0Ri9gHj' - 'tgSXKYeLBgAAAABJRU5ErkJggg==', - 'compressed.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAADAFBM' - 'VEX//////8z//5n//2b//zP//wD/zP//zMz/zJn/zGb/zDP/zAD/' - 'mf//mcz/mZn/mWb/mTP/mQD/Zv//Zsz/Zpn/Zmb/ZjP/ZgD/M///' - 'M8z/M5n/M2b/MzP/MwD/AP//AMz/AJn/AGb/ADP/AADM///M/8zM' - '/5nM/2bM/zPM/wDMzP/MzMzMzJnMzGbMzDPMzADMmf/MmczMmZnM' - 'mWbMmTPMmQDMZv/MZszMZpnMZmbMZjPMZgDMM//MM8zMM5nMM2bM' - 'MzPMMwDMAP/MAMzMAJnMAGbMADPMAACZ//+Z/8yZ/5mZ/2aZ/zOZ' - '/wCZzP+ZzMyZzJmZzGaZzDOZzACZmf+ZmcyZmZmZmWaZmTOZmQCZ' - 'Zv+ZZsyZZpmZZmaZZjOZZgCZM/+ZM8yZM5mZM2aZMzOZMwCZAP+Z' - 'AMyZAJmZAGaZADOZAABm//9m/8xm/5lm/2Zm/zNm/wBmzP9mzMxm' - 'zJlmzGZmzDNmzABmmf9mmcxmmZlmmWZmmTNmmQBmZv9mZsxmZplm' - 'ZmZmZjNmZgBmM/9mM8xmM5lmM2ZmMzNmMwBmAP9mAMxmAJlmAGZm' - 'ADNmAAAz//8z/8wz/5kz/2Yz/zMz/wAzzP8zzMwzzJkzzGYzzDMz' - 'zAAzmf8zmcwzmZkzmWYzmTMzmQAzZv8zZswzZpkzZmYzZjMzZgAz' - 'M/8zM8wzM5kzM2YzMzMzMwAzAP8zAMwzAJkzAGYzADMzAAAA//8A' - '/8wA/5kA/2YA/zMA/wAAzP8AzMwAzJkAzGYAzDMAzAAAmf8AmcwA' - 'mZkAmWYAmTMAmQAAZv8AZswAZpkAZmYAZjMAZgAAM/8AM8wAM5kA' - 'M2YAMzMAMwAAAP8AAMwAAJkAAGYAADPuAADdAAC7AACqAACIAAB3' - 'AABVAABEAAAiAAARAAAA7gAA3QAAuwAAqgAAiAAAdwAAVQAARAAA' - 'IgAAEQAAAO4AAN0AALsAAKoAAIgAAHcAAFUAAEQAACIAABHu7u7d' - '3d27u7uqqqqIiIh3d3dVVVVEREQiIiIREREAAAD7CIKZAAAAJXRS' - 'TlP///////////////////////////////////////////////8A' - 'P89CTwAAAGtJREFUeNp9z9ENgDAIhOEOco+dybVuEXasFMRDY/x5' - '+xJCO6Znu6kSx7BhXyjtKBWWNlwW88Loid7hFRKBXiIYCMfMEYUQ' - 'QohC3CjFA5nIjqx1CqlDLGR/EhM5O06yvin0ftGOyIS7lV14AsQN' - 'aR7rMEBYAAAAAElFTkSuQmCC', - 'folder.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAElBMVEX/' - '////zJnM//+ZZjMzMzMAAADCEvqoAAAAA3RSTlP//wDXyg1BAAAASElE' - 'QVR42s3KQQ6AQAhDUaXt/a/sQDrRJu7c+NmQB0e99B3lnqjT6cYx6zSI' - 'bV40n3D7psYMoBoz4w8/EdNYQsbGEjNxYSljXTEsA9O1pLTvAAAAAElF' - 'TkSuQmCC', - 'text.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAD1BMVEX/' - '///M//+ZmZkzMzMAAABVsTOVAAAAAnRSTlP/AOW3MEoAAABISURBVHja' - 'tcrRCgAgCENRbf7/N7dKomGvngjhMsPLD4NdMPwia438NRIyxsaL/XQZ' - 'hyxpkC6zyjLXGVXnkhqWJWIIrOgeinECLlUCjBCqNQoAAAAASUVORK5C' - 'YII=', - 'unknown.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAD1BMVEX/' - '///M//+ZmZkzMzMAAABVsTOVAAAAAnRSTlP/AOW3MEoAAABYSURBVHja' - 'ncvRDoAgDEPRruX/v1kmNHPBxMTLyzgD6FmsILg56g2hQnJkOco4yZhq' - 'tN5nYd5Zq0LsHblwxwP9GTCWsaGtoelANKzOlz/RfaLYUmLE6E28ALlN' - 'AupSdoFsAAAAAElFTkSuQmCC'} - - -# Begin vendored code -# This code is licensed under the Public Domain/CC0 and comes from -# https://github.com/leenr/gzip-stream/blob/master/gzip_stream.py -# Code was modified: -# removed type annotations to support python2. -# removed use of *, somearg for positional anonymous args. -# Default compression level to 9. - -class GZIPCompressedStream(io.RawIOBase): - def __init__(self, stream, compression_level=9): - assert 1 <= compression_level <= 9 - - self._compression_level = compression_level - self._stream = stream - - self._compressed_stream = io.BytesIO() - self._compressor = gzip.GzipFile( - mode='wb', - fileobj=self._compressed_stream, - compresslevel=compression_level - ) - - # because of the GZIP header written by `GzipFile.__init__`: - self._compressed_stream.seek(0) - - @property - def compression_level(self): - return self._compression_level - - @property - def stream(self): - return self._stream - - def readable(self): - return True - - def _read_compressed_into(self, b): - buf = self._compressed_stream.read(len(b)) - b[:len(buf)] = buf - return len(buf) - - def readinto(self, b): - b = memoryview(b) - - offset = 0 - size = len(b) - while offset < size: - offset += self._read_compressed_into(b[offset:]) - if offset < size: - # self._compressed_buffer now empty - if self._compressor.closed: - # nothing to compress anymore - break - # compress next bytes - self._read_n_compress(size) - - return offset - - def _read_n_compress(self, size): - assert size > 0 - - data = self._stream.read(size) - - # rewind buffer to the start to free up memory - # (because anything currently in the buffer should be already - # streamed off the object) - self._compressed_stream.seek(0) - self._compressed_stream.truncate(0) - - if data: - self._compressor.write(data) - else: - # this will write final data (will flush zlib with Z_FINISH) - self._compressor.close() - - # rewind to the buffer start - self._compressed_stream.seek(0) - - def __repr__(self): - return ( - '{self.__class__.__name__}(' - '{self.stream!r}, ' - 'compression_level={self.compression_level!r}' - ')' - ).format(self=self) - -# End vendored code - - -def get_mime_icon(mime, filename=''): - icon = (APACHE_FILE_ICON_MAP.get(filename) or - APACHE_MIME_ICON_MAP.get(mime) or - APACHE_MIME_ICON_MAP['_default']) - return "data:image/png;base64,%s" % ICON_IMAGES[icon] - - -def retry_function(func): - for attempt in range(1, POST_ATTEMPTS + 1): - try: - return func() - except Exception: - if attempt >= POST_ATTEMPTS: - raise - else: - logging.exception("Error on attempt %d" % attempt) - time.sleep(attempt * 10) - - -def sizeof_fmt(num, suffix='B'): - # From http://stackoverflow.com/questions/1094841/ - # reusable-library-to-get-human-readable-version-of-file-size - for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: - if abs(num) < 1024.0: - return "%3.1f%s%s" % (num, unit, suffix) - num /= 1024.0 - return "%.1f%s%s" % (num, 'Y', suffix) - - -class FileDetail(): - """ - Used to generate indexes with links or as the file path - to push to s3. - """ - - def __init__(self, full_path, relative_path, filename=None): - """ - Args: - full_path (str): The absolute path to the file on disk. - relative_path (str): The relative path from the artifacts source - used for links. - filename (str): An optional alternate filename in links. - """ - # Make FileNotFoundError exception to be compatible with python2 - try: - FileNotFoundError # noqa: F823 - except NameError: - FileNotFoundError = OSError - - self.full_path = full_path - if filename is None: - self.filename = os.path.basename(full_path) - else: - self.filename = filename - self.relative_path = relative_path - - if self.full_path and os.path.isfile(self.full_path): - mime_guess, encoding = mimetypes.guess_type(self.full_path) - self.mimetype = mime_guess if mime_guess else 'text/plain' - self.encoding = encoding - self.folder = False - else: - self.mimetype = 'application/directory' - self.encoding = None - self.folder = True - try: - st = os.stat(self.full_path) - self.last_modified = time.gmtime(st[stat.ST_MTIME]) - self.size = st[stat.ST_SIZE] - except (FileNotFoundError, TypeError): - self.last_modified = time.gmtime(0) - self.size = 0 - - def __repr__(self): - t = 'Folder' if self.folder else 'File' - return '<%s %s>' % (t, self.relative_path) - - -class FileList(Sequence): - '''A collection of FileDetail objects - - This is a list-like group of FileDetail objects, intended to be - used as a context manager around the upload process. - ''' - def __init__(self): - self.file_list = [] - self.file_list.append(FileDetail(None, '', '')) - self.tempdirs = [] - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - for tempdir in self.tempdirs: - shutil.rmtree(tempdir) - - def __getitem__(self, item): - return self.file_list.__getitem__(item) - - def __len__(self): - return self.file_list.__len__() - - def get_tempdir(self): - '''Get a temporary directory - - Returns path to a private temporary directory which will be - cleaned on exit - ''' - tempdir = tempfile.mkdtemp(prefix='s-u-l-tmp') - self.tempdirs.append(tempdir) - return tempdir - - @staticmethod - def _path_in_tree(root, path): - full_path = os.path.realpath(os.path.abspath( - os.path.expanduser(path))) - if not full_path.startswith(root): - logging.debug("Skipping path outside root: %s" % (path,)) - return False - return True - - def add(self, file_path): - """ - Generate a list of files to upload to swift. Recurses through - directories - """ - - # file_list: A list of FileDetails to push to swift - file_list = [] - - if os.path.isfile(file_path): - relative_path = os.path.basename(file_path) - file_list.append(FileDetail(file_path, relative_path)) - elif os.path.isdir(file_path): - original_root = os.path.realpath(os.path.abspath( - os.path.expanduser(file_path))) - - parent_dir = os.path.dirname(file_path) - if not file_path.endswith('/'): - filename = os.path.basename(file_path) - full_path = file_path - relative_name = os.path.relpath(full_path, parent_dir) - file_list.append(FileDetail(full_path, relative_name, - filename)) - # TODO: this will copy the result of symlinked files, but - # it won't follow directory symlinks. If we add that, we - # should ensure that we don't loop. - for path, folders, files in os.walk(file_path): - # Sort folder in-place so that we recurse in order. - files.sort(key=lambda x: x.lower()) - folders.sort(key=lambda x: x.lower()) - # relative_path: The path between the given directory - # and the one being currently walked. - relative_path = os.path.relpath(path, parent_dir) - - for filename in folders: - full_path = os.path.join(path, filename) - if not self._path_in_tree(original_root, full_path): - continue - relative_name = os.path.relpath(full_path, parent_dir) - file_list.append(FileDetail(full_path, relative_name, - filename)) - - for filename in files: - full_path = os.path.join(path, filename) - if not self._path_in_tree(original_root, full_path): - continue - relative_name = os.path.relpath(full_path, parent_dir) - file_detail = FileDetail(full_path, relative_name) - file_list.append(file_detail) - - self.file_list += file_list - - -class Indexer(): - """Index a FileList - - Functions to generate indexes and other collated data for a - FileList - - - make_indexes() : make index.html in folders - """ - def __init__(self, file_list): - ''' - Args: - file_list (FileList): A FileList object with all files - to be indexed. - ''' - assert isinstance(file_list, FileList) - self.file_list = file_list - - def _make_index_file(self, folder_links, title, tempdir, append_footer): - """Writes an index into a file for pushing""" - for file_details in folder_links: - # Do not generate an index file if one exists already. - # This may be the case when uploading other machine generated - # content like python coverage info. - if self.index_filename == file_details.filename: - return - index_content = self._generate_log_index( - folder_links, title, append_footer) - fd = open(os.path.join(tempdir, self.index_filename), 'w') - fd.write(index_content) - return os.path.join(tempdir, self.index_filename) - - def _generate_log_index(self, folder_links, title, append_footer): - """Create an index of logfiles and links to them""" - - output = '%s\n' % title - output += '

%s

\n' % title - output += '' - output += '' - - file_details_to_append = None - for file_details in folder_links: - output += '' - output += ( - '' % ({ - 'm': file_details.mimetype, - 'i': get_mime_icon(file_details.mimetype, - file_details.filename), - })) - filename = file_details.filename - link_filename = filename - if file_details.folder: - filename += '/' - link_filename += '/index.html' - output += '' % ( - urlparse.quote(link_filename), - filename) - output += '' % time.asctime( - file_details.last_modified) - size = sizeof_fmt(file_details.size, suffix='') - output += '' % size - output += '\n' - - if (append_footer and - append_footer in file_details.filename): - file_details_to_append = file_details - - output += '
NameLast ModifiedSize
[ ]%s%s%s
' - - if file_details_to_append: - output += '
' - try: - with open(file_details_to_append.full_path, 'r') as f: - output += f.read() - except IOError: - logging.exception("Error opening file for appending") - - output += '\n' - return output - - def make_indexes(self, create_parent_links=True, - create_topdir_parent_link=False, - append_footer='index_footer.html'): - '''Make index.html files - - Iterate the file list and crete index.html files for folders - - Args: - create_parent_links (bool): Create parent links - create_topdir_parent_link (bool): Create topdir parent link - append_footer (str): Filename of a footer to append to each - generated page - - Return: - No value, the self.file_list will be updated - ''' - self.index_filename = 'index.html' - - folders = collections.OrderedDict() - for f in self.file_list: - if f.folder: - folders[f.relative_path] = [] - folder = os.path.dirname(os.path.dirname( - f.relative_path + '/')) - if folder == '/': - folder = '' - else: - folder = os.path.dirname(f.relative_path) - folders[folder].append(f) - - indexes = {} - parent_file_detail = FileDetail(None, '..', '..') - for folder, files in folders.items(): - # Don't add the pseudo-top-directory - if files and files[0].full_path is None: - files = files[1:] - if create_topdir_parent_link: - files = [parent_file_detail] + files - elif create_parent_links: - files = [parent_file_detail] + files - - # Do generate a link to the parent directory - full_path = self._make_index_file(files, 'Index of %s' % (folder,), - self.file_list.get_tempdir(), - append_footer) - - if full_path: - filename = os.path.basename(full_path) - relative_name = os.path.join(folder, filename) - indexes[folder] = FileDetail(full_path, relative_name) - - # This appends the index file at the end of the group of files - # for each directory. - new_list = [] - last_dirname = None - for f in reversed(list(self.file_list)): - if f.folder: - relative_path = f.relative_path + '/' - else: - relative_path = f.relative_path - dirname = os.path.dirname(relative_path) - if dirname == '/': - dirname = '' - if dirname != last_dirname: - index = indexes.pop(dirname, None) - if index: - new_list.append(index) - last_dirname = dirname - new_list.append(f) - new_list.reverse() - self.file_list.file_list = new_list - - -class GzipFilter(): - chunk_size = 16384 - - def __init__(self, infile): - self.gzipfile = GZIPCompressedStream(infile) - self.done = False - - def __iter__(self): - return self - - def __next__(self): - if self.done: - self.gzipfile.close() - raise StopIteration() - data = self.gzipfile.read(self.chunk_size) - if not data: - self.done = True - return data - - -class DeflateFilter(): - chunk_size = 16384 - - def __init__(self, infile): - self.infile = infile - self.encoder = zlib.compressobj() - self.done = False - - def __iter__(self): - return self - - def __next__(self): - if self.done: - raise StopIteration() - ret = b'' - while True: - data = self.infile.read(self.chunk_size) - if data: - ret = self.encoder.compress(data) - if ret: - break - else: - self.done = True - ret = self.encoder.flush() - break - return ret class Uploader(): diff --git a/roles/upload-logs-base/library/zuul_swift_upload.py b/roles/upload-logs-base/library/zuul_swift_upload.py index 51dcc43ee..bfa34bcaa 100755 --- a/roles/upload-logs-base/library/zuul_swift_upload.py +++ b/roles/upload-logs-base/library/zuul_swift_upload.py @@ -25,28 +25,15 @@ Utility to upload files to swift """ import argparse -import gzip -import io import logging -import mimetypes import os try: import queue as queuelib except ImportError: import Queue as queuelib -import shutil -import stat import sys -import tempfile import threading -import time import traceback -try: - import urllib.parse as urlparse -except ImportError: - import urllib as urlparse -import zlib -import collections import openstack import requests @@ -57,176 +44,23 @@ import keystoneauth1.exceptions from ansible.module_utils.basic import AnsibleModule try: - # Python 3.3+ - from collections.abc import Sequence + # Ansible context + from ansible.module_utils.zuul_jobs.upload_utils import ( + FileList, + GzipFilter, + Indexer, + retry_function, + ) except ImportError: - from collections import Sequence - -mimetypes.init() -mimetypes.add_type('text/plain', '.yaml') + # Test context + from ..module_utils.zuul_jobs.upload_utils import ( + FileList, + GzipFilter, + Indexer, + retry_function, + ) MAX_UPLOAD_THREADS = 24 -POST_ATTEMPTS = 3 - -# Map mime types to apache icons -APACHE_MIME_ICON_MAP = { - '_default': 'unknown.png', - 'application/gzip': 'compressed.png', - 'application/directory': 'folder.png', - 'text/html': 'text.png', - 'text/plain': 'text.png', -} - -# Map mime types to apache icons -APACHE_FILE_ICON_MAP = { - '..': 'back.png', -} - -# These icon files are from the Apache project and are in the public -# domain. -ICON_IMAGES = { - 'back.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAElBMVEX/' - '///M//+ZmZlmZmYzMzMAAACei5rnAAAAAnRSTlP/AOW3MEoAAABWSURB' - 'VHjabdBBCgAhDEPRRpv7X3kwEMsQ//IRRC08urjRHbha5VLFUsVSxVI9' - 'lmDh5hMpHD6n0EgoiZG0DNINpnWlcVXaRix76e1/8dddcL6nG0Ri9gHj' - 'tgSXKYeLBgAAAABJRU5ErkJggg==', - 'compressed.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAADAFBM' - 'VEX//////8z//5n//2b//zP//wD/zP//zMz/zJn/zGb/zDP/zAD/' - 'mf//mcz/mZn/mWb/mTP/mQD/Zv//Zsz/Zpn/Zmb/ZjP/ZgD/M///' - 'M8z/M5n/M2b/MzP/MwD/AP//AMz/AJn/AGb/ADP/AADM///M/8zM' - '/5nM/2bM/zPM/wDMzP/MzMzMzJnMzGbMzDPMzADMmf/MmczMmZnM' - 'mWbMmTPMmQDMZv/MZszMZpnMZmbMZjPMZgDMM//MM8zMM5nMM2bM' - 'MzPMMwDMAP/MAMzMAJnMAGbMADPMAACZ//+Z/8yZ/5mZ/2aZ/zOZ' - '/wCZzP+ZzMyZzJmZzGaZzDOZzACZmf+ZmcyZmZmZmWaZmTOZmQCZ' - 'Zv+ZZsyZZpmZZmaZZjOZZgCZM/+ZM8yZM5mZM2aZMzOZMwCZAP+Z' - 'AMyZAJmZAGaZADOZAABm//9m/8xm/5lm/2Zm/zNm/wBmzP9mzMxm' - 'zJlmzGZmzDNmzABmmf9mmcxmmZlmmWZmmTNmmQBmZv9mZsxmZplm' - 'ZmZmZjNmZgBmM/9mM8xmM5lmM2ZmMzNmMwBmAP9mAMxmAJlmAGZm' - 'ADNmAAAz//8z/8wz/5kz/2Yz/zMz/wAzzP8zzMwzzJkzzGYzzDMz' - 'zAAzmf8zmcwzmZkzmWYzmTMzmQAzZv8zZswzZpkzZmYzZjMzZgAz' - 'M/8zM8wzM5kzM2YzMzMzMwAzAP8zAMwzAJkzAGYzADMzAAAA//8A' - '/8wA/5kA/2YA/zMA/wAAzP8AzMwAzJkAzGYAzDMAzAAAmf8AmcwA' - 'mZkAmWYAmTMAmQAAZv8AZswAZpkAZmYAZjMAZgAAM/8AM8wAM5kA' - 'M2YAMzMAMwAAAP8AAMwAAJkAAGYAADPuAADdAAC7AACqAACIAAB3' - 'AABVAABEAAAiAAARAAAA7gAA3QAAuwAAqgAAiAAAdwAAVQAARAAA' - 'IgAAEQAAAO4AAN0AALsAAKoAAIgAAHcAAFUAAEQAACIAABHu7u7d' - '3d27u7uqqqqIiIh3d3dVVVVEREQiIiIREREAAAD7CIKZAAAAJXRS' - 'TlP///////////////////////////////////////////////8A' - 'P89CTwAAAGtJREFUeNp9z9ENgDAIhOEOco+dybVuEXasFMRDY/x5' - '+xJCO6Znu6kSx7BhXyjtKBWWNlwW88Loid7hFRKBXiIYCMfMEYUQ' - 'QohC3CjFA5nIjqx1CqlDLGR/EhM5O06yvin0ftGOyIS7lV14AsQN' - 'aR7rMEBYAAAAAElFTkSuQmCC', - 'folder.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAElBMVEX/' - '////zJnM//+ZZjMzMzMAAADCEvqoAAAAA3RSTlP//wDXyg1BAAAASElE' - 'QVR42s3KQQ6AQAhDUaXt/a/sQDrRJu7c+NmQB0e99B3lnqjT6cYx6zSI' - 'bV40n3D7psYMoBoz4w8/EdNYQsbGEjNxYSljXTEsA9O1pLTvAAAAAElF' - 'TkSuQmCC', - 'text.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAD1BMVEX/' - '///M//+ZmZkzMzMAAABVsTOVAAAAAnRSTlP/AOW3MEoAAABISURBVHja' - 'tcrRCgAgCENRbf7/N7dKomGvngjhMsPLD4NdMPwia438NRIyxsaL/XQZ' - 'hyxpkC6zyjLXGVXnkhqWJWIIrOgeinECLlUCjBCqNQoAAAAASUVORK5C' - 'YII=', - 'unknown.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAD1BMVEX/' - '///M//+ZmZkzMzMAAABVsTOVAAAAAnRSTlP/AOW3MEoAAABYSURBVHja' - 'ncvRDoAgDEPRruX/v1kmNHPBxMTLyzgD6FmsILg56g2hQnJkOco4yZhq' - 'tN5nYd5Zq0LsHblwxwP9GTCWsaGtoelANKzOlz/RfaLYUmLE6E28ALlN' - 'AupSdoFsAAAAAElFTkSuQmCC'} - - -# Begin vendored code -# This code is licensed under the Public Domain/CC0 and comes from -# https://github.com/leenr/gzip-stream/blob/master/gzip_stream.py -# Code was modified: -# removed type annotations to support python2. -# removed use of *, somearg for positional anonymous args. -# Default compression level to 9. - -class GZIPCompressedStream(io.RawIOBase): - def __init__(self, stream, compression_level=9): - assert 1 <= compression_level <= 9 - - self._compression_level = compression_level - self._stream = stream - - self._compressed_stream = io.BytesIO() - self._compressor = gzip.GzipFile( - mode='wb', - fileobj=self._compressed_stream, - compresslevel=compression_level - ) - - # because of the GZIP header written by `GzipFile.__init__`: - self._compressed_stream.seek(0) - - @property - def compression_level(self): - return self._compression_level - - @property - def stream(self): - return self._stream - - def readable(self): - return True - - def _read_compressed_into(self, b): - buf = self._compressed_stream.read(len(b)) - b[:len(buf)] = buf - return len(buf) - - def readinto(self, b): - b = memoryview(b) - - offset = 0 - size = len(b) - while offset < size: - offset += self._read_compressed_into(b[offset:]) - if offset < size: - # self._compressed_buffer now empty - if self._compressor.closed: - # nothing to compress anymore - break - # compress next bytes - self._read_n_compress(size) - - return offset - - def _read_n_compress(self, size): - assert size > 0 - - data = self._stream.read(size) - - # rewind buffer to the start to free up memory - # (because anything currently in the buffer should be already - # streamed off the object) - self._compressed_stream.seek(0) - self._compressed_stream.truncate(0) - - if data: - self._compressor.write(data) - else: - # this will write final data (will flush zlib with Z_FINISH) - self._compressor.close() - - # rewind to the buffer start - self._compressed_stream.seek(0) - - def __repr__(self): - return ( - '{self.__class__.__name__}(' - '{self.stream!r}, ' - 'compression_level={self.compression_level!r}' - ')' - ).format(self=self) - -# End vendored code - - -def get_mime_icon(mime, filename=''): - icon = (APACHE_FILE_ICON_MAP.get(filename) or - APACHE_MIME_ICON_MAP.get(mime) or - APACHE_MIME_ICON_MAP['_default']) - return "data:image/png;base64,%s" % ICON_IMAGES[icon] def get_cloud(cloud): @@ -237,371 +71,6 @@ def get_cloud(cloud): return openstack.connect(cloud=cloud) -def retry_function(func): - for attempt in range(1, POST_ATTEMPTS + 1): - try: - return func() - except Exception: - if attempt >= POST_ATTEMPTS: - raise - else: - logging.exception("Error on attempt %d" % attempt) - time.sleep(attempt * 10) - - -def sizeof_fmt(num, suffix='B'): - # From http://stackoverflow.com/questions/1094841/ - # reusable-library-to-get-human-readable-version-of-file-size - for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: - if abs(num) < 1024.0: - return "%3.1f%s%s" % (num, unit, suffix) - num /= 1024.0 - return "%.1f%s%s" % (num, 'Y', suffix) - - -class FileDetail(): - """ - Used to generate indexes with links or as the file path - to push to swift. - """ - - def __init__(self, full_path, relative_path, filename=None): - """ - Args: - full_path (str): The absolute path to the file on disk. - relative_path (str): The relative path from the artifacts source - used for links. - filename (str): An optional alternate filename in links. - """ - # Make FileNotFoundError exception to be compatible with python2 - try: - FileNotFoundError # noqa: F823 - except NameError: - FileNotFoundError = OSError - - self.full_path = full_path - if filename is None: - self.filename = os.path.basename(full_path) - else: - self.filename = filename - self.relative_path = relative_path - - if self.full_path and os.path.isfile(self.full_path): - mime_guess, encoding = mimetypes.guess_type(self.full_path) - self.mimetype = mime_guess if mime_guess else 'text/plain' - self.encoding = encoding - self.folder = False - else: - self.mimetype = 'application/directory' - self.encoding = None - self.folder = True - try: - st = os.stat(self.full_path) - self.last_modified = time.gmtime(st[stat.ST_MTIME]) - self.size = st[stat.ST_SIZE] - except (FileNotFoundError, TypeError): - self.last_modified = time.gmtime(0) - self.size = 0 - - def __repr__(self): - t = 'Folder' if self.folder else 'File' - return '<%s %s>' % (t, self.relative_path) - - -class FileList(Sequence): - '''A collection of FileDetail objects - - This is a list-like group of FileDetail objects, intended to be - used as a context manager around the upload process. - ''' - def __init__(self): - self.file_list = [] - self.file_list.append(FileDetail(None, '', '')) - self.tempdirs = [] - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - for tempdir in self.tempdirs: - shutil.rmtree(tempdir) - - def __getitem__(self, item): - return self.file_list.__getitem__(item) - - def __len__(self): - return self.file_list.__len__() - - def get_tempdir(self): - '''Get a temporary directory - - Returns path to a private temporary directory which will be - cleaned on exit - ''' - tempdir = tempfile.mkdtemp(prefix='s-u-l-tmp') - self.tempdirs.append(tempdir) - return tempdir - - @staticmethod - def _path_in_tree(root, path): - full_path = os.path.realpath(os.path.abspath( - os.path.expanduser(path))) - if not full_path.startswith(root): - logging.debug("Skipping path outside root: %s" % (path,)) - return False - return True - - def add(self, file_path): - """ - Generate a list of files to upload to swift. Recurses through - directories - """ - - # file_list: A list of FileDetails to push to swift - file_list = [] - - if os.path.isfile(file_path): - relative_path = os.path.basename(file_path) - file_list.append(FileDetail(file_path, relative_path)) - elif os.path.isdir(file_path): - original_root = os.path.realpath(os.path.abspath( - os.path.expanduser(file_path))) - - parent_dir = os.path.dirname(file_path) - if not file_path.endswith('/'): - filename = os.path.basename(file_path) - full_path = file_path - relative_name = os.path.relpath(full_path, parent_dir) - file_list.append(FileDetail(full_path, relative_name, - filename)) - # TODO: this will copy the result of symlinked files, but - # it won't follow directory symlinks. If we add that, we - # should ensure that we don't loop. - for path, folders, files in os.walk(file_path): - # Sort folder in-place so that we recurse in order. - files.sort(key=lambda x: x.lower()) - folders.sort(key=lambda x: x.lower()) - # relative_path: The path between the given directory - # and the one being currently walked. - relative_path = os.path.relpath(path, parent_dir) - - for filename in folders: - full_path = os.path.join(path, filename) - if not self._path_in_tree(original_root, full_path): - continue - relative_name = os.path.relpath(full_path, parent_dir) - file_list.append(FileDetail(full_path, relative_name, - filename)) - - for filename in files: - full_path = os.path.join(path, filename) - if not self._path_in_tree(original_root, full_path): - continue - relative_name = os.path.relpath(full_path, parent_dir) - file_detail = FileDetail(full_path, relative_name) - file_list.append(file_detail) - - self.file_list += file_list - - -class Indexer(): - """Index a FileList - - Functions to generate indexes and other collated data for a - FileList - - - make_indexes() : make index.html in folders - """ - def __init__(self, file_list): - ''' - Args: - file_list (FileList): A FileList object with all files - to be indexed. - ''' - assert isinstance(file_list, FileList) - self.file_list = file_list - - def _make_index_file(self, folder_links, title, tempdir, append_footer): - """Writes an index into a file for pushing""" - for file_details in folder_links: - # Do not generate an index file if one exists already. - # This may be the case when uploading other machine generated - # content like python coverage info. - if self.index_filename == file_details.filename: - return - index_content = self._generate_log_index( - folder_links, title, append_footer) - fd = open(os.path.join(tempdir, self.index_filename), 'w') - fd.write(index_content) - return os.path.join(tempdir, self.index_filename) - - def _generate_log_index(self, folder_links, title, append_footer): - """Create an index of logfiles and links to them""" - - output = '%s\n' % title - output += '

%s

\n' % title - output += '' - output += '' - - file_details_to_append = None - for file_details in folder_links: - output += '' - output += ( - '' % ({ - 'm': file_details.mimetype, - 'i': get_mime_icon(file_details.mimetype, - file_details.filename), - })) - filename = file_details.filename - if file_details.folder: - filename += '/' - output += '' % ( - urlparse.quote(filename), - filename) - output += '' % time.asctime( - file_details.last_modified) - size = sizeof_fmt(file_details.size, suffix='') - output += '' % size - output += '\n' - - if (append_footer and - append_footer in file_details.filename): - file_details_to_append = file_details - - output += '
NameLast ModifiedSize
[ ]%s%s%s
' - - if file_details_to_append: - output += '
' - try: - with open(file_details_to_append.full_path, 'r') as f: - output += f.read() - except IOError: - logging.exception("Error opening file for appending") - - output += '\n' - return output - - def make_indexes(self, create_parent_links=True, - create_topdir_parent_link=False, - append_footer='index_footer.html'): - '''Make index.html files - - Iterate the file list and crete index.html files for folders - - Args: - create_parent_links (bool): Create parent links - create_topdir_parent_link (bool): Create topdir parent link - append_footer (str): Filename of a footer to append to each - generated page - - Return: - No value, the self.file_list will be updated - ''' - self.index_filename = 'index.html' - - folders = collections.OrderedDict() - for f in self.file_list: - if f.folder: - folders[f.relative_path] = [] - folder = os.path.dirname(os.path.dirname( - f.relative_path + '/')) - if folder == '/': - folder = '' - else: - folder = os.path.dirname(f.relative_path) - folders[folder].append(f) - - indexes = {} - parent_file_detail = FileDetail(None, '..', '..') - for folder, files in folders.items(): - # Don't add the pseudo-top-directory - if files and files[0].full_path is None: - files = files[1:] - if create_topdir_parent_link: - files = [parent_file_detail] + files - elif create_parent_links: - files = [parent_file_detail] + files - - # Do generate a link to the parent directory - full_path = self._make_index_file(files, 'Index of %s' % (folder,), - self.file_list.get_tempdir(), - append_footer) - - if full_path: - filename = os.path.basename(full_path) - relative_name = os.path.join(folder, filename) - indexes[folder] = FileDetail(full_path, relative_name) - - # This appends the index file at the end of the group of files - # for each directory. - new_list = [] - last_dirname = None - for f in reversed(list(self.file_list)): - if f.folder: - relative_path = f.relative_path + '/' - else: - relative_path = f.relative_path - dirname = os.path.dirname(relative_path) - if dirname == '/': - dirname = '' - if dirname != last_dirname: - index = indexes.pop(dirname, None) - if index: - new_list.append(index) - last_dirname = dirname - new_list.append(f) - new_list.reverse() - self.file_list.file_list = new_list - - -class GzipFilter(): - chunk_size = 16384 - - def __init__(self, infile): - self.gzipfile = GZIPCompressedStream(infile) - self.done = False - - def __iter__(self): - return self - - def __next__(self): - if self.done: - self.gzipfile.close() - raise StopIteration() - data = self.gzipfile.read(self.chunk_size) - if not data: - self.done = True - return data - - -class DeflateFilter(): - chunk_size = 16384 - - def __init__(self, infile): - self.infile = infile - self.encoder = zlib.compressobj() - self.done = False - - def __iter__(self): - return self - - def __next__(self): - if self.done: - raise StopIteration() - ret = b'' - while True: - data = self.infile.read(self.chunk_size) - if data: - ret = self.encoder.compress(data) - if ret: - break - else: - self.done = True - ret = self.encoder.flush() - break - return ret - - class Uploader(): def __init__(self, cloud, container, prefix=None, delete_after=None, public=True, dry_run=False): diff --git a/roles/upload-logs-base/module_utils/__init__.py b/roles/upload-logs-base/module_utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/roles/upload-logs-base/module_utils/zuul_jobs/__init__.py b/roles/upload-logs-base/module_utils/zuul_jobs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/roles/upload-logs-base/module_utils/zuul_jobs/upload_utils.py b/roles/upload-logs-base/module_utils/zuul_jobs/upload_utils.py new file mode 100644 index 000000000..c9d34f302 --- /dev/null +++ b/roles/upload-logs-base/module_utils/zuul_jobs/upload_utils.py @@ -0,0 +1,608 @@ +#!/usr/bin/env python3 +# +# Copyright 2014 Rackspace Australia +# Copyright 2018-2019 Red Hat, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +""" +Generic utilities used for log upload. +""" + +import datetime +import gzip +import io +import json +import logging +import mimetypes +import os +import shutil +import stat +import tempfile +import time +try: + import urllib.parse as urlparse +except ImportError: + import urllib as urlparse +import zlib +import collections + +import google.auth.compute_engine.credentials as gce_cred + + +try: + # Python 3.3+ + from collections.abc import Sequence +except ImportError: + from collections import Sequence + +mimetypes.init() +mimetypes.add_type('text/plain', '.yaml') + +MAX_UPLOAD_THREADS = 24 +POST_ATTEMPTS = 3 + +# Map mime types to apache icons +APACHE_MIME_ICON_MAP = { + '_default': 'unknown.png', + 'application/gzip': 'compressed.png', + 'application/directory': 'folder.png', + 'text/html': 'text.png', + 'text/plain': 'text.png', +} + +# Map mime types to apache icons +APACHE_FILE_ICON_MAP = { + '..': 'back.png', +} + +# These icon files are from the Apache project and are in the public +# domain. +ICON_IMAGES = { + 'back.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAElBMVEX/' + '///M//+ZmZlmZmYzMzMAAACei5rnAAAAAnRSTlP/AOW3MEoAAABWSURB' + 'VHjabdBBCgAhDEPRRpv7X3kwEMsQ//IRRC08urjRHbha5VLFUsVSxVI9' + 'lmDh5hMpHD6n0EgoiZG0DNINpnWlcVXaRix76e1/8dddcL6nG0Ri9gHj' + 'tgSXKYeLBgAAAABJRU5ErkJggg==', + 'compressed.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAADAFBM' + 'VEX//////8z//5n//2b//zP//wD/zP//zMz/zJn/zGb/zDP/zAD/' + 'mf//mcz/mZn/mWb/mTP/mQD/Zv//Zsz/Zpn/Zmb/ZjP/ZgD/M///' + 'M8z/M5n/M2b/MzP/MwD/AP//AMz/AJn/AGb/ADP/AADM///M/8zM' + '/5nM/2bM/zPM/wDMzP/MzMzMzJnMzGbMzDPMzADMmf/MmczMmZnM' + 'mWbMmTPMmQDMZv/MZszMZpnMZmbMZjPMZgDMM//MM8zMM5nMM2bM' + 'MzPMMwDMAP/MAMzMAJnMAGbMADPMAACZ//+Z/8yZ/5mZ/2aZ/zOZ' + '/wCZzP+ZzMyZzJmZzGaZzDOZzACZmf+ZmcyZmZmZmWaZmTOZmQCZ' + 'Zv+ZZsyZZpmZZmaZZjOZZgCZM/+ZM8yZM5mZM2aZMzOZMwCZAP+Z' + 'AMyZAJmZAGaZADOZAABm//9m/8xm/5lm/2Zm/zNm/wBmzP9mzMxm' + 'zJlmzGZmzDNmzABmmf9mmcxmmZlmmWZmmTNmmQBmZv9mZsxmZplm' + 'ZmZmZjNmZgBmM/9mM8xmM5lmM2ZmMzNmMwBmAP9mAMxmAJlmAGZm' + 'ADNmAAAz//8z/8wz/5kz/2Yz/zMz/wAzzP8zzMwzzJkzzGYzzDMz' + 'zAAzmf8zmcwzmZkzmWYzmTMzmQAzZv8zZswzZpkzZmYzZjMzZgAz' + 'M/8zM8wzM5kzM2YzMzMzMwAzAP8zAMwzAJkzAGYzADMzAAAA//8A' + '/8wA/5kA/2YA/zMA/wAAzP8AzMwAzJkAzGYAzDMAzAAAmf8AmcwA' + 'mZkAmWYAmTMAmQAAZv8AZswAZpkAZmYAZjMAZgAAM/8AM8wAM5kA' + 'M2YAMzMAMwAAAP8AAMwAAJkAAGYAADPuAADdAAC7AACqAACIAAB3' + 'AABVAABEAAAiAAARAAAA7gAA3QAAuwAAqgAAiAAAdwAAVQAARAAA' + 'IgAAEQAAAO4AAN0AALsAAKoAAIgAAHcAAFUAAEQAACIAABHu7u7d' + '3d27u7uqqqqIiIh3d3dVVVVEREQiIiIREREAAAD7CIKZAAAAJXRS' + 'TlP///////////////////////////////////////////////8A' + 'P89CTwAAAGtJREFUeNp9z9ENgDAIhOEOco+dybVuEXasFMRDY/x5' + '+xJCO6Znu6kSx7BhXyjtKBWWNlwW88Loid7hFRKBXiIYCMfMEYUQ' + 'QohC3CjFA5nIjqx1CqlDLGR/EhM5O06yvin0ftGOyIS7lV14AsQN' + 'aR7rMEBYAAAAAElFTkSuQmCC', + 'folder.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAElBMVEX/' + '////zJnM//+ZZjMzMzMAAADCEvqoAAAAA3RSTlP//wDXyg1BAAAASElE' + 'QVR42s3KQQ6AQAhDUaXt/a/sQDrRJu7c+NmQB0e99B3lnqjT6cYx6zSI' + 'bV40n3D7psYMoBoz4w8/EdNYQsbGEjNxYSljXTEsA9O1pLTvAAAAAElF' + 'TkSuQmCC', + 'text.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAD1BMVEX/' + '///M//+ZmZkzMzMAAABVsTOVAAAAAnRSTlP/AOW3MEoAAABISURBVHja' + 'tcrRCgAgCENRbf7/N7dKomGvngjhMsPLD4NdMPwia438NRIyxsaL/XQZ' + 'hyxpkC6zyjLXGVXnkhqWJWIIrOgeinECLlUCjBCqNQoAAAAASUVORK5C' + 'YII=', + 'unknown.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAD1BMVEX/' + '///M//+ZmZkzMzMAAABVsTOVAAAAAnRSTlP/AOW3MEoAAABYSURBVHja' + 'ncvRDoAgDEPRruX/v1kmNHPBxMTLyzgD6FmsILg56g2hQnJkOco4yZhq' + 'tN5nYd5Zq0LsHblwxwP9GTCWsaGtoelANKzOlz/RfaLYUmLE6E28ALlN' + 'AupSdoFsAAAAAElFTkSuQmCC'} + + +# Begin vendored code +# This code is licensed under the Public Domain/CC0 and comes from +# https://github.com/leenr/gzip-stream/blob/master/gzip_stream.py +# Code was modified: +# removed type annotations to support python2. +# removed use of *, somearg for positional anonymous args. +# Default compression level to 9. + +class GZIPCompressedStream(io.RawIOBase): + def __init__(self, stream, compression_level=9): + assert 1 <= compression_level <= 9 + + self._compression_level = compression_level + self._stream = stream + + self._compressed_stream = io.BytesIO() + self._compressor = gzip.GzipFile( + mode='wb', + fileobj=self._compressed_stream, + compresslevel=compression_level + ) + + # because of the GZIP header written by `GzipFile.__init__`: + self._compressed_stream.seek(0) + self.count = 0 + + def read(self, length): + r = super().read(length) + self.count += len(r) + return r + + def tell(self): + return self.count + + @property + def compression_level(self): + return self._compression_level + + @property + def stream(self): + return self._stream + + def readable(self): + return True + + def _read_compressed_into(self, b): + buf = self._compressed_stream.read(len(b)) + b[:len(buf)] = buf + return len(buf) + + def readinto(self, b): + b = memoryview(b) + + offset = 0 + size = len(b) + while offset < size: + offset += self._read_compressed_into(b[offset:]) + if offset < size: + # self._compressed_buffer now empty + if self._compressor.closed: + # nothing to compress anymore + break + # compress next bytes + self._read_n_compress(size) + + return offset + + def _read_n_compress(self, size): + assert size > 0 + + data = self._stream.read(size) + + # rewind buffer to the start to free up memory + # (because anything currently in the buffer should be already + # streamed off the object) + self._compressed_stream.seek(0) + self._compressed_stream.truncate(0) + + if data: + self._compressor.write(data) + else: + # this will write final data (will flush zlib with Z_FINISH) + self._compressor.close() + + # rewind to the buffer start + self._compressed_stream.seek(0) + + def __repr__(self): + return ( + '{self.__class__.__name__}(' + '{self.stream!r}, ' + 'compression_level={self.compression_level!r}' + ')' + ).format(self=self) + +# End vendored code + + +def get_mime_icon(mime, filename=''): + icon = (APACHE_FILE_ICON_MAP.get(filename) or + APACHE_MIME_ICON_MAP.get(mime) or + APACHE_MIME_ICON_MAP['_default']) + return "data:image/png;base64,%s" % ICON_IMAGES[icon] + + +def retry_function(func): + for attempt in range(1, POST_ATTEMPTS + 1): + try: + return func() + except Exception: + if attempt >= POST_ATTEMPTS: + raise + else: + logging.exception("Error on attempt %d" % attempt) + time.sleep(attempt * 10) + + +def sizeof_fmt(num, suffix='B'): + # From http://stackoverflow.com/questions/1094841/ + # reusable-library-to-get-human-readable-version-of-file-size + for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: + if abs(num) < 1024.0: + return "%3.1f%s%s" % (num, unit, suffix) + num /= 1024.0 + return "%.1f%s%s" % (num, 'Y', suffix) + + +class Credentials(gce_cred.Credentials): + def __init__(self, path): + super(Credentials, self).__init__() + self._path = path + self.refresh(None) + + def refresh(self, request): + with open(self._path) as f: + data = json.loads(f.read()) + self.token = data['access_token'] + self.expiry = (datetime.datetime.utcnow() + + datetime.timedelta(seconds=data['expires_in'])) + + +class FileDetail(): + """ + Used to generate indexes with links or as the file path + to push to storage. + """ + + def __init__(self, full_path, relative_path, filename=None): + """ + Args: + full_path (str): The absolute path to the file on disk. + relative_path (str): The relative path from the artifacts source + used for links. + filename (str): An optional alternate filename in links. + """ + # Make FileNotFoundError exception to be compatible with python2 + try: + FileNotFoundError # noqa: F823 + except NameError: + FileNotFoundError = OSError + + self.full_path = full_path + if filename is None: + self.filename = os.path.basename(full_path) + else: + self.filename = filename + self.relative_path = relative_path + + if self.full_path and os.path.isfile(self.full_path): + mime_guess, encoding = mimetypes.guess_type(self.full_path) + self.mimetype = mime_guess if mime_guess else 'text/plain' + self.encoding = encoding + self.folder = False + else: + self.mimetype = 'application/directory' + self.encoding = None + self.folder = True + try: + st = os.stat(self.full_path) + self.last_modified = time.gmtime(st[stat.ST_MTIME]) + self.size = st[stat.ST_SIZE] + except (FileNotFoundError, TypeError): + self.last_modified = time.gmtime(0) + self.size = 0 + + def __repr__(self): + t = 'Folder' if self.folder else 'File' + return '<%s %s>' % (t, self.relative_path) + + +class FileList(Sequence): + '''A collection of FileDetail objects + + This is a list-like group of FileDetail objects, intended to be + used as a context manager around the upload process. + ''' + def __init__(self): + self.file_list = [] + self.file_list.append(FileDetail(None, '', '')) + self.tempdirs = [] + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + for tempdir in self.tempdirs: + shutil.rmtree(tempdir) + + def __getitem__(self, item): + return self.file_list.__getitem__(item) + + def __len__(self): + return self.file_list.__len__() + + def get_tempdir(self): + '''Get a temporary directory + + Returns path to a private temporary directory which will be + cleaned on exit + ''' + tempdir = tempfile.mkdtemp(prefix='s-u-l-tmp') + self.tempdirs.append(tempdir) + return tempdir + + @staticmethod + def _path_in_tree(root, path): + full_path = os.path.realpath(os.path.abspath( + os.path.expanduser(path))) + if not full_path.startswith(root): + logging.debug("Skipping path outside root: %s" % (path,)) + return False + return True + + def add(self, file_path): + """ + Generate a list of files to upload to storage. Recurses through + directories + """ + + # file_list: A list of FileDetails to push to storage + file_list = [] + + if os.path.isfile(file_path): + relative_path = os.path.basename(file_path) + file_list.append(FileDetail(file_path, relative_path)) + elif os.path.isdir(file_path): + original_root = os.path.realpath(os.path.abspath( + os.path.expanduser(file_path))) + + parent_dir = os.path.dirname(file_path) + if not file_path.endswith('/'): + filename = os.path.basename(file_path) + full_path = file_path + relative_name = os.path.relpath(full_path, parent_dir) + file_list.append(FileDetail(full_path, relative_name, + filename)) + # TODO: this will copy the result of symlinked files, but + # it won't follow directory symlinks. If we add that, we + # should ensure that we don't loop. + for path, folders, files in os.walk(file_path): + # Sort folder in-place so that we recurse in order. + files.sort(key=lambda x: x.lower()) + folders.sort(key=lambda x: x.lower()) + # relative_path: The path between the given directory + # and the one being currently walked. + relative_path = os.path.relpath(path, parent_dir) + + for filename in folders: + full_path = os.path.join(path, filename) + if not self._path_in_tree(original_root, full_path): + continue + relative_name = os.path.relpath(full_path, parent_dir) + file_list.append(FileDetail(full_path, relative_name, + filename)) + + for filename in files: + full_path = os.path.join(path, filename) + if not self._path_in_tree(original_root, full_path): + continue + relative_name = os.path.relpath(full_path, parent_dir) + file_detail = FileDetail(full_path, relative_name) + file_list.append(file_detail) + + self.file_list += file_list + + +class Indexer(): + """Index a FileList + + Functions to generate indexes and other collated data for a + FileList + + - make_indexes() : make index.html in folders + """ + def __init__(self, file_list): + ''' + Args: + file_list (FileList): A FileList object with all files + to be indexed. + ''' + assert isinstance(file_list, FileList) + self.file_list = file_list + + def _make_index_file(self, folder_links, title, tempdir, append_footer): + """Writes an index into a file for pushing""" + for file_details in folder_links: + # Do not generate an index file if one exists already. + # This may be the case when uploading other machine generated + # content like python coverage info. + if self.index_filename == file_details.filename: + return + index_content = self._generate_log_index( + folder_links, title, append_footer) + fd = open(os.path.join(tempdir, self.index_filename), 'w') + fd.write(index_content) + return os.path.join(tempdir, self.index_filename) + + def _generate_log_index(self, folder_links, title, append_footer): + """Create an index of logfiles and links to them""" + + output = '%s\n' % title + output += '

%s

\n' % title + output += '' + output += '' + + file_details_to_append = None + for file_details in folder_links: + output += '' + output += ( + '' % ({ + 'm': file_details.mimetype, + 'i': get_mime_icon(file_details.mimetype, + file_details.filename), + })) + filename = file_details.filename + link_filename = filename + if file_details.folder: + filename += '/' + link_filename += '/index.html' + output += '' % ( + urlparse.quote(link_filename), + filename) + output += '' % time.asctime( + file_details.last_modified) + size = sizeof_fmt(file_details.size, suffix='') + output += '' % size + output += '\n' + + if (append_footer and + append_footer in file_details.filename): + file_details_to_append = file_details + + output += '
NameLast ModifiedSize
[ ]%s%s%s
' + + if file_details_to_append: + output += '
' + try: + with open(file_details_to_append.full_path, 'r') as f: + output += f.read() + except IOError: + logging.exception("Error opening file for appending") + + output += '\n' + return output + + def make_indexes(self, create_parent_links=True, + create_topdir_parent_link=False, + append_footer='index_footer.html'): + '''Make index.html files + + Iterate the file list and crete index.html files for folders + + Args: + create_parent_links (bool): Create parent links + create_topdir_parent_link (bool): Create topdir parent link + append_footer (str): Filename of a footer to append to each + generated page + + Return: + No value, the self.file_list will be updated + ''' + self.index_filename = 'index.html' + + folders = collections.OrderedDict() + for f in self.file_list: + if f.folder: + folders[f.relative_path] = [] + folder = os.path.dirname(os.path.dirname( + f.relative_path + '/')) + if folder == '/': + folder = '' + else: + folder = os.path.dirname(f.relative_path) + folders[folder].append(f) + + indexes = {} + parent_file_detail = FileDetail(None, '..', '..') + for folder, files in folders.items(): + # Don't add the pseudo-top-directory + if files and files[0].full_path is None: + files = files[1:] + if create_topdir_parent_link: + files = [parent_file_detail] + files + elif create_parent_links: + files = [parent_file_detail] + files + + # Do generate a link to the parent directory + full_path = self._make_index_file(files, 'Index of %s' % (folder,), + self.file_list.get_tempdir(), + append_footer) + + if full_path: + filename = os.path.basename(full_path) + relative_name = os.path.join(folder, filename) + indexes[folder] = FileDetail(full_path, relative_name) + + # This appends the index file at the end of the group of files + # for each directory. + new_list = [] + last_dirname = None + for f in reversed(list(self.file_list)): + if f.folder: + relative_path = f.relative_path + '/' + else: + relative_path = f.relative_path + dirname = os.path.dirname(relative_path) + if dirname == '/': + dirname = '' + if dirname != last_dirname: + index = indexes.pop(dirname, None) + if index: + new_list.append(index) + last_dirname = dirname + new_list.append(f) + new_list.reverse() + self.file_list.file_list = new_list + + +class GzipFilter(): + chunk_size = 16384 + + def __init__(self, infile): + self.gzipfile = GZIPCompressedStream(infile) + self.done = False + + def __iter__(self): + return self + + def __next__(self): + if self.done: + self.gzipfile.close() + raise StopIteration() + data = self.gzipfile.read(self.chunk_size) + if not data: + self.done = True + return data + + +class DeflateFilter(): + chunk_size = 16384 + + def __init__(self, infile): + self.infile = infile + self.encoder = zlib.compressobj() + self.done = False + + def __iter__(self): + return self + + def __next__(self): + if self.done: + raise StopIteration() + ret = b'' + while True: + data = self.infile.read(self.chunk_size) + if data: + ret = self.encoder.compress(data) + if ret: + break + else: + self.done = True + ret = self.encoder.flush() + break + return ret