zuul/zuul-jobs
Code Issues Proposed changes

Merge "Use urlencoded filenames in test fixtures"

Browse Source
This commit is contained in:
Zuul
2021-02-01 14:11:44 +00:00
committed by Gerrit Code Review
parent 79c44ead3a dfbc8e2ec8
commit 5bb91e867b
11 changed files with 734 additions and 406 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
roles/generate-zuul-manifest/library/test-fixtures/.gitattributes vendored Normal file
View File

@@ -0,0 +1 @@
*.* eol=lf

139
roles/test-upload-logs-swift/library/filefixture.py Normal file
View File

@@ -0,0 +1,139 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Handle file name special characters in a file tree.
All files stored in a filetree can be renamed to urlencoded filenames.
A file tree can also be copied to a temporary location with file names
decoded, to be used in tests with special characters that are not always
possible to store on all file systems.
"""
from __future__ import print_function
import os
try:
from urllib.parse import quote as urlib_quote
from urllib.parse import unquote as urlib_unquote
except ImportError:
from urllib import quote as urlib_quote
from urllib import unquote as urlib_unquote
import argparse
import fixtures
import tempfile
import shutil
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
'test-fixtures')
SAFE_CHARS = "\\/"
def portable_makedirs_exist_ok(path):
try:
os.makedirs(path, exist_ok=True)
except TypeError as err:
if "unexpected keyword argument" not in str(err):
raise err
if not os.path.exists(path):
try:
os.makedirs(path)
except OSError as err:
if "File exists" not in err:
raise err
def urlencode_filetree():
for root, _, files in os.walk(FIXTURE_DIR):
for filename in files:
os.rename(
os.path.join(root, filename),
os.path.join(
root, urlib_quote(urlib_unquote(filename), SAFE_CHARS)
)
)
def populate_filetree(dst_dir=None):
if not os.path.exists(FIXTURE_DIR):
return None
if not dst_dir:
dst_dir = tempfile.mkdtemp()
portable_makedirs_exist_ok(dst_dir)
for root, dirs, files in os.walk(FIXTURE_DIR):
dst_root = root.replace(FIXTURE_DIR, dst_dir, 1)
for directory in dirs:
portable_makedirs_exist_ok(os.path.join(dst_root, directory))
for filename in files:
try:
shutil.copyfile(
os.path.join(root, filename),
os.path.join(dst_root, urlib_unquote(filename))
)
except IOError as err:
print(
"\nFile {}".format(
os.path.join(dst_root, urlib_unquote(filename))
),
"\nnot possible to write to disk,",
"\npossibly due to filename not being valid on Windows?\n"
)
shutil.rmtree(dst_dir)
raise err
return dst_dir
class FileFixture(fixtures.Fixture):
def _setUp(self):
self.root = tempfile.mkdtemp()
self.addCleanup(self.local_clean_up)
populate_filetree(self.root)
# There is no cleanup action, as the filetree is left intact for other
# tests to use
def local_clean_up(self):
shutil.rmtree(self.root)
if __name__ == '__main__':
parser = argparse.ArgumentParser(__doc__)
parser.add_argument(
'--populate',
help="Causes files in {}".format(FIXTURE_DIR) +
"to be copied with decoded file name to a tmp dir" +
"Overrides --encode",
action='store_true'
)
parser.add_argument(
'--encode',
help="Causes files under {} to be renamed with urlencoding.".format(
FIXTURE_DIR
) + "DEFAULT behaviour, overridden by --populate",
action='store_true'
)
args = parser.parse_args()
if args.populate:
print(populate_filetree())
else:
urlencode_filetree()

1
roles/test-upload-logs-swift/library/test-fixtures/.gitattributes vendored Normal file
View File

@@ -0,0 +1 @@
*.* eol=lf

0
roles/test-upload-logs-swift/library/test-fixtures/logs/Ꮓບບξ-unicode.txt → roles/test-upload-logs-swift/library/test-fixtures/logs/%E1%8F%83%E0%BA%9A%E0%BA%9A%CE%BE-unicode.txt
View File

0
roles/test-upload-logs-swift/library/test-fixtures/logs/controller/subdir/foo::3.txt → roles/test-upload-logs-swift/library/test-fixtures/logs/controller/subdir/foo%3A%3A3.txt
View File

422
roles/test-upload-logs-swift/library/test_zuul_swift_upload.py
View File

@@ -32,7 +32,7 @@ except ImportError:
import requests import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from .zuul_swift_upload import FileList, Indexer, FileDetail, Uploader from .zuul_swift_upload import FileList, Indexer, FileDetail, Uploader
from .filefixture import FileFixture
FIXTURE_DIR = os.path.join(os.path.dirname(__file__), FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
'test-fixtures') 'test-fixtures')
@@ -50,8 +50,11 @@ class SymlinkFixture(fixtures.Fixture):
] ]
def _setUp(self): def _setUp(self):
self.file_fixture = FileFixture()
self.file_fixture.setUp()
self.addCleanup(self.file_fixture.cleanUp)
for (src, target) in self.links: for (src, target) in self.links:
path = os.path.join(FIXTURE_DIR, 'links', src) path = os.path.join(self.file_fixture.root, 'links', src)
os.symlink(target, path) os.symlink(target, path)
self.addCleanup(os.unlink, path) self.addCleanup(os.unlink, path)
@@ -88,66 +91,69 @@ class TestFileList(testtools.TestCase):
'''Test a single directory with a trailing slash''' '''Test a single directory with a trailing slash'''
with FileList() as fl: with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs/')) with FileFixture() as file_fixture:
self.assert_files(fl, [ fl.add(os.path.join(file_fixture.root, 'logs/'))
('', 'application/directory', None), self.assert_files(fl, [
('controller', 'application/directory', None), ('', 'application/directory', None),
('zuul-info', 'application/directory', None), ('controller', 'application/directory', None),
('job-output.json', 'application/json', None), ('zuul-info', 'application/directory', None),
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', ('job-output.json', 'application/json', None),
'text/plain', None), (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
('controller/subdir', 'application/directory', None), 'text/plain', None),
('controller/compressed.gz', 'text/plain', 'gzip'), ('controller/subdir', 'application/directory', None),
('controller/cpu-load.svg', 'image/svg+xml', None), ('controller/compressed.gz', 'text/plain', 'gzip'),
('controller/journal.xz', 'text/plain', 'xz'), ('controller/cpu-load.svg', 'image/svg+xml', None),
('controller/service_log.txt', 'text/plain', None), ('controller/journal.xz', 'text/plain', 'xz'),
('controller/syslog', 'text/plain', None), ('controller/service_log.txt', 'text/plain', None),
('controller/subdir/foo::3.txt', 'text/plain', None), ('controller/syslog', 'text/plain', None),
('controller/subdir/subdir.txt', 'text/plain', None), ('controller/subdir/foo::3.txt', 'text/plain', None),
('zuul-info/inventory.yaml', 'text/plain', None), ('controller/subdir/subdir.txt', 'text/plain', None),
('zuul-info/zuul-info.controller.txt', 'text/plain', None), ('zuul-info/inventory.yaml', 'text/plain', None),
]) ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
])
def test_single_dir(self): def test_single_dir(self):
'''Test a single directory without a trailing slash''' '''Test a single directory without a trailing slash'''
with FileList() as fl: with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs')) with FileFixture() as file_fixture:
self.assert_files(fl, [ fl.add(os.path.join(file_fixture.root, 'logs'))
('', 'application/directory', None), self.assert_files(fl, [
('logs', 'application/directory', None), ('', 'application/directory', None),
('logs/controller', 'application/directory', None), ('logs', 'application/directory', None),
('logs/zuul-info', 'application/directory', None), ('logs/controller', 'application/directory', None),
('logs/job-output.json', 'application/json', None), ('logs/zuul-info', 'application/directory', None),
(u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt', ('logs/job-output.json', 'application/json', None),
'text/plain', None), (u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
('logs/controller/subdir', 'application/directory', None), 'text/plain', None),
('logs/controller/compressed.gz', 'text/plain', 'gzip'), ('logs/controller/subdir', 'application/directory', None),
('logs/controller/cpu-load.svg', 'image/svg+xml', None), ('logs/controller/compressed.gz', 'text/plain', 'gzip'),
('logs/controller/journal.xz', 'text/plain', 'xz'), ('logs/controller/cpu-load.svg', 'image/svg+xml', None),
('logs/controller/service_log.txt', 'text/plain', None), ('logs/controller/journal.xz', 'text/plain', 'xz'),
('logs/controller/syslog', 'text/plain', None), ('logs/controller/service_log.txt', 'text/plain', None),
('logs/controller/subdir/foo::3.txt', 'text/plain', None), ('logs/controller/syslog', 'text/plain', None),
('logs/controller/subdir/subdir.txt', 'text/plain', None), ('logs/controller/subdir/foo::3.txt', 'text/plain', None),
('logs/zuul-info/inventory.yaml', 'text/plain', None), ('logs/controller/subdir/subdir.txt', 'text/plain', None),
('logs/zuul-info/zuul-info.controller.txt', ('logs/zuul-info/inventory.yaml', 'text/plain', None),
'text/plain', None), ('logs/zuul-info/zuul-info.controller.txt',
]) 'text/plain', None),
])
def test_single_file(self): def test_single_file(self):
'''Test a single file''' '''Test a single file'''
with FileList() as fl: with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, with FileFixture() as file_fixture:
'logs/zuul-info/inventory.yaml')) fl.add(os.path.join(file_fixture.root,
self.assert_files(fl, [ 'logs/zuul-info/inventory.yaml'))
('', 'application/directory', None), self.assert_files(fl, [
('inventory.yaml', 'text/plain', None), ('', 'application/directory', None),
]) ('inventory.yaml', 'text/plain', None),
])
def test_symlinks(self): def test_symlinks(self):
'''Test symlinks''' '''Test symlinks'''
with FileList() as fl: with FileList() as fl:
self.useFixture(SymlinkFixture()) symlink_fixture = self.useFixture(SymlinkFixture())
fl.add(os.path.join(FIXTURE_DIR, 'links/')) fl.add(os.path.join(symlink_fixture.file_fixture.root, 'links/'))
self.assert_files(fl, [ self.assert_files(fl, [
('', 'application/directory', None), ('', 'application/directory', None),
('controller', 'application/directory', None), ('controller', 'application/directory', None),
@@ -165,7 +171,8 @@ class TestFileList(testtools.TestCase):
def test_index_files(self): def test_index_files(self):
'''Test index generation''' '''Test index generation'''
with FileList() as fl: with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs')) symlink_fixture = self.useFixture(SymlinkFixture())
fl.add(os.path.join(symlink_fixture.file_fixture.root, 'logs'))
ix = Indexer(fl) ix = Indexer(fl)
ix.make_indexes() ix.make_indexes()
@@ -223,32 +230,33 @@ class TestFileList(testtools.TestCase):
def test_index_files_trailing_slash(self): def test_index_files_trailing_slash(self):
'''Test index generation with a trailing slash''' '''Test index generation with a trailing slash'''
with FileList() as fl: with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs/')) with FileFixture() as file_fixture:
ix = Indexer(fl) fl.add(os.path.join(file_fixture.root, 'logs/'))
ix.make_indexes() ix = Indexer(fl)
ix.make_indexes()
self.assert_files(fl, [ self.assert_files(fl, [
('', 'application/directory', None), ('', 'application/directory', None),
('controller', 'application/directory', None), ('controller', 'application/directory', None),
('zuul-info', 'application/directory', None), ('zuul-info', 'application/directory', None),
('job-output.json', 'application/json', None), ('job-output.json', 'application/json', None),
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
'text/plain', None), 'text/plain', None),
('index.html', 'text/html', None), ('index.html', 'text/html', None),
('controller/subdir', 'application/directory', None), ('controller/subdir', 'application/directory', None),
('controller/compressed.gz', 'text/plain', 'gzip'), ('controller/compressed.gz', 'text/plain', 'gzip'),
('controller/cpu-load.svg', 'image/svg+xml', None), ('controller/cpu-load.svg', 'image/svg+xml', None),
('controller/journal.xz', 'text/plain', 'xz'), ('controller/journal.xz', 'text/plain', 'xz'),
('controller/service_log.txt', 'text/plain', None), ('controller/service_log.txt', 'text/plain', None),
('controller/syslog', 'text/plain', None), ('controller/syslog', 'text/plain', None),
('controller/index.html', 'text/html', None), ('controller/index.html', 'text/html', None),
('controller/subdir/foo::3.txt', 'text/plain', None), ('controller/subdir/foo::3.txt', 'text/plain', None),
('controller/subdir/subdir.txt', 'text/plain', None), ('controller/subdir/subdir.txt', 'text/plain', None),
('controller/subdir/index.html', 'text/html', None), ('controller/subdir/index.html', 'text/html', None),
('zuul-info/inventory.yaml', 'text/plain', None), ('zuul-info/inventory.yaml', 'text/plain', None),
('zuul-info/zuul-info.controller.txt', 'text/plain', None), ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
('zuul-info/index.html', 'text/html', None), ('zuul-info/index.html', 'text/html', None),
]) ])
top_index = self.find_file(fl, 'index.html') top_index = self.find_file(fl, 'index.html')
page = open(top_index.full_path).read() page = open(top_index.full_path).read()
@@ -280,134 +288,145 @@ class TestFileList(testtools.TestCase):
def test_topdir_parent_link(self): def test_topdir_parent_link(self):
'''Test index generation creates topdir parent link''' '''Test index generation creates topdir parent link'''
with FileList() as fl: with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs/')) with FileFixture() as file_fixture:
ix = Indexer(fl) fl.add(os.path.join(file_fixture.root, 'logs/'))
ix.make_indexes( ix = Indexer(fl)
create_parent_links=True, ix.make_indexes(
create_topdir_parent_link=True) create_parent_links=True,
create_topdir_parent_link=True)
self.assert_files(fl, [ self.assert_files(fl, [
('', 'application/directory', None), ('', 'application/directory', None),
('controller', 'application/directory', None), ('controller', 'application/directory', None),
('zuul-info', 'application/directory', None), ('zuul-info', 'application/directory', None),
('job-output.json', 'application/json', None), ('job-output.json', 'application/json', None),
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
'text/plain', None), 'text/plain', None),
('index.html', 'text/html', None), ('index.html', 'text/html', None),
('controller/subdir', 'application/directory', None), ('controller/subdir', 'application/directory', None),
('controller/compressed.gz', 'text/plain', 'gzip'), ('controller/compressed.gz', 'text/plain', 'gzip'),
('controller/cpu-load.svg', 'image/svg+xml', None), ('controller/cpu-load.svg', 'image/svg+xml', None),
('controller/journal.xz', 'text/plain', 'xz'), ('controller/journal.xz', 'text/plain', 'xz'),
('controller/service_log.txt', 'text/plain', None), ('controller/service_log.txt', 'text/plain', None),
('controller/syslog', 'text/plain', None), ('controller/syslog', 'text/plain', None),
('controller/index.html', 'text/html', None), ('controller/index.html', 'text/html', None),
('controller/subdir/foo::3.txt', 'text/plain', None), ('controller/subdir/foo::3.txt', 'text/plain', None),
('controller/subdir/subdir.txt', 'text/plain', None), ('controller/subdir/subdir.txt', 'text/plain', None),
('controller/subdir/index.html', 'text/html', None), ('controller/subdir/index.html', 'text/html', None),
('zuul-info/inventory.yaml', 'text/plain', None), ('zuul-info/inventory.yaml', 'text/plain', None),
('zuul-info/zuul-info.controller.txt', 'text/plain', None), ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
('zuul-info/index.html', 'text/html', None), ('zuul-info/index.html', 'text/html', None),
]) ])
top_index = self.find_file(fl, 'index.html') top_index = self.find_file(fl, 'index.html')
page = open(top_index.full_path).read() page = open(top_index.full_path).read()
page = BeautifulSoup(page, 'html.parser') page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:] rows = page.find_all('tr')[1:]
self.assertEqual(len(rows), 5) self.assertEqual(len(rows), 5)
self.assertEqual(rows[0].find('a').get('href'), '../') self.assertEqual(rows[0].find('a').get('href'), '../')
self.assertEqual(rows[0].find('a').text, '../') self.assertEqual(rows[0].find('a').text, '../')
self.assertEqual(rows[1].find('a').get('href'), 'controller/') self.assertEqual(rows[1].find('a').get('href'), 'controller/')
self.assertEqual(rows[1].find('a').text, 'controller/') self.assertEqual(rows[1].find('a').text, 'controller/')
self.assertEqual(rows[2].find('a').get('href'), 'zuul-info/') self.assertEqual(rows[2].find('a').get('href'), 'zuul-info/')
self.assertEqual(rows[2].find('a').text, 'zuul-info/') self.assertEqual(rows[2].find('a').text, 'zuul-info/')
subdir_index = self.find_file(fl, 'controller/subdir/index.html') subdir_index = self.find_file(
page = open(subdir_index.full_path).read() fl, 'controller/subdir/index.html'
page = BeautifulSoup(page, 'html.parser') )
rows = page.find_all('tr')[1:] page = open(subdir_index.full_path).read()
self.assertEqual(rows[0].find('a').get('href'), '../') page = BeautifulSoup(page, 'html.parser')
self.assertEqual(rows[0].find('a').text, '../') rows = page.find_all('tr')[1:]
self.assertEqual(rows[0].find('a').get('href'), '../')
self.assertEqual(rows[0].find('a').text, '../')
# Test proper escaping of files with funny names # Test proper escaping of files with funny names
self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt') self.assertEqual(
self.assertEqual(rows[1].find('a').text, 'foo::3.txt') rows[1].find('a').get('href'), 'foo%3A%3A3.txt'
# Test files without escaping )
self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt') self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
self.assertEqual(rows[2].find('a').text, 'subdir.txt') # Test files without escaping
self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
self.assertEqual(rows[2].find('a').text, 'subdir.txt')
def test_no_parent_links(self): def test_no_parent_links(self):
'''Test index generation creates topdir parent link''' '''Test index generation creates topdir parent link'''
with FileList() as fl: with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs/')) with FileFixture() as file_fixture:
ix = Indexer(fl) fl.add(os.path.join(file_fixture.root, 'logs/'))
ix.make_indexes( ix = Indexer(fl)
create_parent_links=False, ix.make_indexes(
create_topdir_parent_link=False) create_parent_links=False,
create_topdir_parent_link=False)
self.assert_files(fl, [ self.assert_files(fl, [
('', 'application/directory', None), ('', 'application/directory', None),
('controller', 'application/directory', None), ('controller', 'application/directory', None),
('zuul-info', 'application/directory', None), ('zuul-info', 'application/directory', None),
('job-output.json', 'application/json', None), ('job-output.json', 'application/json', None),
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
'text/plain', None), 'text/plain', None),
('index.html', 'text/html', None), ('index.html', 'text/html', None),
('controller/subdir', 'application/directory', None), ('controller/subdir', 'application/directory', None),
('controller/compressed.gz', 'text/plain', 'gzip'), ('controller/compressed.gz', 'text/plain', 'gzip'),
('controller/cpu-load.svg', 'image/svg+xml', None), ('controller/cpu-load.svg', 'image/svg+xml', None),
('controller/journal.xz', 'text/plain', 'xz'), ('controller/journal.xz', 'text/plain', 'xz'),
('controller/service_log.txt', 'text/plain', None), ('controller/service_log.txt', 'text/plain', None),
('controller/syslog', 'text/plain', None), ('controller/syslog', 'text/plain', None),
('controller/index.html', 'text/html', None), ('controller/index.html', 'text/html', None),
('controller/subdir/foo::3.txt', 'text/plain', None), ('controller/subdir/foo::3.txt', 'text/plain', None),
('controller/subdir/subdir.txt', 'text/plain', None), ('controller/subdir/subdir.txt', 'text/plain', None),
('controller/subdir/index.html', 'text/html', None), ('controller/subdir/index.html', 'text/html', None),
('zuul-info/inventory.yaml', 'text/plain', None), ('zuul-info/inventory.yaml', 'text/plain', None),
('zuul-info/zuul-info.controller.txt', 'text/plain', None), ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
('zuul-info/index.html', 'text/html', None), ('zuul-info/index.html', 'text/html', None),
]) ])
top_index = self.find_file(fl, 'index.html') top_index = self.find_file(fl, 'index.html')
page = open(top_index.full_path).read() page = open(top_index.full_path).read()
page = BeautifulSoup(page, 'html.parser') page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:] rows = page.find_all('tr')[1:]
self.assertEqual(len(rows), 4) self.assertEqual(len(rows), 4)
self.assertEqual(rows[0].find('a').get('href'), 'controller/') self.assertEqual(rows[0].find('a').get('href'), 'controller/')
self.assertEqual(rows[0].find('a').text, 'controller/') self.assertEqual(rows[0].find('a').text, 'controller/')
self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/') self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
self.assertEqual(rows[1].find('a').text, 'zuul-info/') self.assertEqual(rows[1].find('a').text, 'zuul-info/')
subdir_index = self.find_file(fl, 'controller/subdir/index.html') subdir_index = self.find_file(
page = open(subdir_index.full_path).read() fl, 'controller/subdir/index.html'
page = BeautifulSoup(page, 'html.parser') )
rows = page.find_all('tr')[1:] page = open(subdir_index.full_path).read()
page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:]
# Test proper escaping of files with funny names # Test proper escaping of files with funny names
self.assertEqual(rows[0].find('a').get('href'), 'foo%3A%3A3.txt') self.assertEqual(
self.assertEqual(rows[0].find('a').text, 'foo::3.txt') rows[0].find('a').get('href'), 'foo%3A%3A3.txt'
# Test files without escaping )
self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt') self.assertEqual(rows[0].find('a').text, 'foo::3.txt')
self.assertEqual(rows[1].find('a').text, 'subdir.txt') # Test files without escaping
self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
self.assertEqual(rows[1].find('a').text, 'subdir.txt')
class TestFileDetail(testtools.TestCase): class TestFileDetail(testtools.TestCase):
def test_get_file_detail(self): def test_get_file_detail(self):
'''Test files info''' '''Test files info'''
path = os.path.join(FIXTURE_DIR, 'logs/job-output.json') with FileFixture() as file_fixture:
file_detail = FileDetail(path, '') path = os.path.join(file_fixture.root, 'logs/job-output.json')
path_stat = os.stat(path) file_detail = FileDetail(path, '')
self.assertEqual( path_stat = os.stat(path)
time.gmtime(path_stat[stat.ST_MTIME]), self.assertEqual(
file_detail.last_modified) time.gmtime(path_stat[stat.ST_MTIME]),
self.assertEqual(16, file_detail.size) file_detail.last_modified)
self.assertEqual(16, file_detail.size)
def test_get_file_detail_missing_file(self): def test_get_file_detail_missing_file(self):
'''Test files that go missing during a walk''' '''Test files that go missing during a walk'''
@@ -447,26 +466,29 @@ class TestUpload(testtools.TestCase):
) )
# Get some test files to upload # Get some test files to upload
files = [ with FileFixture() as file_fixture:
FileDetail( files = [
os.path.join(FIXTURE_DIR, "logs/job-output.json"), FileDetail(
"job-output.json", os.path.join(file_fixture.root, "logs/job-output.json"),
), "job-output.json",
FileDetail(
os.path.join(FIXTURE_DIR, "logs/zuul-info/inventory.yaml"),
"inventory.yaml",
),
]
expected_failures = [
{
"file": "job-output.json",
"error": (
"Error posting file after multiple attempts: "
"Failed for a reason"
), ),
}, FileDetail(
] os.path.join(
file_fixture.root, "logs/zuul-info/inventory.yaml"
),
"inventory.yaml",
),
]
failures = uploader.upload(files) expected_failures = [
self.assertEqual(expected_failures, failures) {
"file": "job-output.json",
"error": (
"Error posting file after multiple attempts: "
"Failed for a reason"
),
},
]
failures = uploader.upload(files)
self.assertEqual(expected_failures, failures)

139
roles/upload-logs-base/library/filefixture.py Normal file
View File

@@ -0,0 +1,139 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Handle file name special characters in a file tree.
All files stored in a filetree can be renamed to urlencoded filenames.
A file tree can also be copied to a temporary location with file names
decoded, to be used in tests with special characters that are not always
possible to store on all file systems.
"""
from __future__ import print_function
import os
try:
from urllib.parse import quote as urlib_quote
from urllib.parse import unquote as urlib_unquote
except ImportError:
from urllib import quote as urlib_quote
from urllib import unquote as urlib_unquote
import argparse
import fixtures
import tempfile
import shutil
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
'test-fixtures')
SAFE_CHARS = "\\/"
def portable_makedirs_exist_ok(path):
try:
os.makedirs(path, exist_ok=True)
except TypeError as err:
if "unexpected keyword argument" not in str(err):
raise err
if not os.path.exists(path):
try:
os.makedirs(path)
except OSError as err:
if "File exists" not in err:
raise err
def urlencode_filetree():
for root, _, files in os.walk(FIXTURE_DIR):
for filename in files:
os.rename(
os.path.join(root, filename),
os.path.join(
root, urlib_quote(urlib_unquote(filename), SAFE_CHARS)
)
)
def populate_filetree(dst_dir=None):
if not os.path.exists(FIXTURE_DIR):
return None
if not dst_dir:
dst_dir = tempfile.mkdtemp()
portable_makedirs_exist_ok(dst_dir)
for root, dirs, files in os.walk(FIXTURE_DIR):
dst_root = root.replace(FIXTURE_DIR, dst_dir, 1)
for directory in dirs:
portable_makedirs_exist_ok(os.path.join(dst_root, directory))
for filename in files:
try:
shutil.copyfile(
os.path.join(root, filename),
os.path.join(dst_root, urlib_unquote(filename))
)
except IOError as err:
print(
"\nFile {}".format(
os.path.join(dst_root, urlib_unquote(filename))
),
"\nnot possible to write to disk,",
"\npossibly due to filename not being valid on Windows?\n"
)
shutil.rmtree(dst_dir)
raise err
return dst_dir
class FileFixture(fixtures.Fixture):
def _setUp(self):
self.root = tempfile.mkdtemp()
self.addCleanup(self.local_clean_up)
populate_filetree(self.root)
# There is no cleanup action, as the filetree is left intact for other
# tests to use
def local_clean_up(self):
shutil.rmtree(self.root)
if __name__ == '__main__':
parser = argparse.ArgumentParser(__doc__)
parser.add_argument(
'--populate',
help="Causes files in {}".format(FIXTURE_DIR) +
"to be copied with decoded file name to a tmp dir" +
"Overrides --encode",
action='store_true'
)
parser.add_argument(
'--encode',
help="Causes files under {} to be renamed with urlencoding.".format(
FIXTURE_DIR
) + "DEFAULT behaviour, overridden by --populate",
action='store_true'
)
args = parser.parse_args()
if args.populate:
print(populate_filetree())
else:
urlencode_filetree()

1
roles/upload-logs-base/library/test-fixtures/.gitattributes vendored Normal file
View File

@@ -0,0 +1 @@
*.* eol=lf

0
roles/upload-logs-base/library/test-fixtures/logs/Ꮓບບξ-unicode.txt → roles/upload-logs-base/library/test-fixtures/logs/%E1%8F%83%E0%BA%9A%E0%BA%9A%CE%BE-unicode.txt
View File

0
roles/upload-logs-base/library/test-fixtures/logs/controller/subdir/foo::3.txt → roles/upload-logs-base/library/test-fixtures/logs/controller/subdir/foo%3A%3A3.txt
View File

437
roles/upload-logs-base/library/test_index.py
View File

@@ -33,6 +33,7 @@ import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from .zuul_swift_upload import Uploader from .zuul_swift_upload import Uploader
from ..module_utils.zuul_jobs.upload_utils import FileList, Indexer, FileDetail from ..module_utils.zuul_jobs.upload_utils import FileList, Indexer, FileDetail
from .filefixture import FileFixture
FIXTURE_DIR = os.path.join(os.path.dirname(__file__), FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
'test-fixtures') 'test-fixtures')
@@ -50,8 +51,11 @@ class SymlinkFixture(fixtures.Fixture):
] ]
def _setUp(self): def _setUp(self):
self.file_fixture = FileFixture()
self.file_fixture.setUp()
self.addCleanup(self.file_fixture.cleanUp)
for (src, target) in self.links: for (src, target) in self.links:
path = os.path.join(FIXTURE_DIR, 'links', src) path = os.path.join(self.file_fixture.root, 'links', src)
os.symlink(target, path) os.symlink(target, path)
self.addCleanup(os.unlink, path) self.addCleanup(os.unlink, path)
@@ -88,66 +92,69 @@ class TestFileList(testtools.TestCase):
'''Test a single directory with a trailing slash''' '''Test a single directory with a trailing slash'''
with FileList() as fl: with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs/')) with FileFixture() as file_fixture:
self.assert_files(fl, [ fl.add(os.path.join(file_fixture.root, 'logs/'))
('', 'application/directory', None), self.assert_files(fl, [
('controller', 'application/directory', None), ('', 'application/directory', None),
('zuul-info', 'application/directory', None), ('controller', 'application/directory', None),
('job-output.json', 'application/json', None), ('zuul-info', 'application/directory', None),
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', ('job-output.json', 'application/json', None),
'text/plain', None), (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
('controller/subdir', 'application/directory', None), 'text/plain', None),
('controller/compressed.gz', 'text/plain', 'gzip'), ('controller/subdir', 'application/directory', None),
('controller/cpu-load.svg', 'image/svg+xml', None), ('controller/compressed.gz', 'text/plain', 'gzip'),
('controller/journal.xz', 'text/plain', 'xz'), ('controller/cpu-load.svg', 'image/svg+xml', None),
('controller/service_log.txt', 'text/plain', None), ('controller/journal.xz', 'text/plain', 'xz'),
('controller/syslog', 'text/plain', None), ('controller/service_log.txt', 'text/plain', None),
('controller/subdir/foo::3.txt', 'text/plain', None), ('controller/syslog', 'text/plain', None),
('controller/subdir/subdir.txt', 'text/plain', None), ('controller/subdir/foo::3.txt', 'text/plain', None),
('zuul-info/inventory.yaml', 'text/plain', None), ('controller/subdir/subdir.txt', 'text/plain', None),
('zuul-info/zuul-info.controller.txt', 'text/plain', None), ('zuul-info/inventory.yaml', 'text/plain', None),
]) ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
])
def test_single_dir(self): def test_single_dir(self):
'''Test a single directory without a trailing slash''' '''Test a single directory without a trailing slash'''
with FileList() as fl: with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs')) with FileFixture() as file_fixture:
self.assert_files(fl, [ fl.add(os.path.join(file_fixture.root, 'logs'))
('', 'application/directory', None), self.assert_files(fl, [
('logs', 'application/directory', None), ('', 'application/directory', None),
('logs/controller', 'application/directory', None), ('logs', 'application/directory', None),
('logs/zuul-info', 'application/directory', None), ('logs/controller', 'application/directory', None),
('logs/job-output.json', 'application/json', None), ('logs/zuul-info', 'application/directory', None),
(u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt', ('logs/job-output.json', 'application/json', None),
'text/plain', None), (u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
('logs/controller/subdir', 'application/directory', None), 'text/plain', None),
('logs/controller/compressed.gz', 'text/plain', 'gzip'), ('logs/controller/subdir', 'application/directory', None),
('logs/controller/cpu-load.svg', 'image/svg+xml', None), ('logs/controller/compressed.gz', 'text/plain', 'gzip'),
('logs/controller/journal.xz', 'text/plain', 'xz'), ('logs/controller/cpu-load.svg', 'image/svg+xml', None),
('logs/controller/service_log.txt', 'text/plain', None), ('logs/controller/journal.xz', 'text/plain', 'xz'),
('logs/controller/syslog', 'text/plain', None), ('logs/controller/service_log.txt', 'text/plain', None),
('logs/controller/subdir/foo::3.txt', 'text/plain', None), ('logs/controller/syslog', 'text/plain', None),
('logs/controller/subdir/subdir.txt', 'text/plain', None), ('logs/controller/subdir/foo::3.txt', 'text/plain', None),
('logs/zuul-info/inventory.yaml', 'text/plain', None), ('logs/controller/subdir/subdir.txt', 'text/plain', None),
('logs/zuul-info/zuul-info.controller.txt', ('logs/zuul-info/inventory.yaml', 'text/plain', None),
'text/plain', None), ('logs/zuul-info/zuul-info.controller.txt',
]) 'text/plain', None),
])
def test_single_file(self): def test_single_file(self):
'''Test a single file''' '''Test a single file'''
with FileList() as fl: with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, with FileFixture() as file_fixture:
'logs/zuul-info/inventory.yaml')) fl.add(os.path.join(file_fixture.root,
self.assert_files(fl, [ 'logs/zuul-info/inventory.yaml'))
('', 'application/directory', None), self.assert_files(fl, [
('inventory.yaml', 'text/plain', None), ('', 'application/directory', None),
]) ('inventory.yaml', 'text/plain', None),
])
def test_symlinks(self): def test_symlinks(self):
'''Test symlinks''' '''Test symlinks'''
with FileList() as fl: with FileList() as fl:
self.useFixture(SymlinkFixture()) symlink_fixture = self.useFixture(SymlinkFixture())
fl.add(os.path.join(FIXTURE_DIR, 'links/')) fl.add(os.path.join(symlink_fixture.file_fixture.root, 'links/'))
self.assert_files(fl, [ self.assert_files(fl, [
('', 'application/directory', None), ('', 'application/directory', None),
('controller', 'application/directory', None), ('controller', 'application/directory', None),
@@ -165,7 +172,8 @@ class TestFileList(testtools.TestCase):
def test_index_files(self): def test_index_files(self):
'''Test index generation''' '''Test index generation'''
with FileList() as fl: with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs')) symlink_fixture = self.useFixture(SymlinkFixture())
fl.add(os.path.join(symlink_fixture.file_fixture.root, 'logs'))
ix = Indexer(fl) ix = Indexer(fl)
ix.make_indexes() ix.make_indexes()
@@ -223,32 +231,33 @@ class TestFileList(testtools.TestCase):
def test_index_files_trailing_slash(self): def test_index_files_trailing_slash(self):
'''Test index generation with a trailing slash''' '''Test index generation with a trailing slash'''
with FileList() as fl: with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs/')) with FileFixture() as file_fixture:
ix = Indexer(fl) fl.add(os.path.join(file_fixture.root, 'logs/'))
ix.make_indexes() ix = Indexer(fl)
ix.make_indexes()
self.assert_files(fl, [ self.assert_files(fl, [
('', 'application/directory', None), ('', 'application/directory', None),
('controller', 'application/directory', None), ('controller', 'application/directory', None),
('zuul-info', 'application/directory', None), ('zuul-info', 'application/directory', None),
('job-output.json', 'application/json', None), ('job-output.json', 'application/json', None),
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
'text/plain', None), 'text/plain', None),
('index.html', 'text/html', None), ('index.html', 'text/html', None),
('controller/subdir', 'application/directory', None), ('controller/subdir', 'application/directory', None),
('controller/compressed.gz', 'text/plain', 'gzip'), ('controller/compressed.gz', 'text/plain', 'gzip'),
('controller/cpu-load.svg', 'image/svg+xml', None), ('controller/cpu-load.svg', 'image/svg+xml', None),
('controller/journal.xz', 'text/plain', 'xz'), ('controller/journal.xz', 'text/plain', 'xz'),
('controller/service_log.txt', 'text/plain', None), ('controller/service_log.txt', 'text/plain', None),
('controller/syslog', 'text/plain', None), ('controller/syslog', 'text/plain', None),
('controller/index.html', 'text/html', None), ('controller/index.html', 'text/html', None),
('controller/subdir/foo::3.txt', 'text/plain', None), ('controller/subdir/foo::3.txt', 'text/plain', None),
('controller/subdir/subdir.txt', 'text/plain', None), ('controller/subdir/subdir.txt', 'text/plain', None),
('controller/subdir/index.html', 'text/html', None), ('controller/subdir/index.html', 'text/html', None),
('zuul-info/inventory.yaml', 'text/plain', None), ('zuul-info/inventory.yaml', 'text/plain', None),
('zuul-info/zuul-info.controller.txt', 'text/plain', None), ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
('zuul-info/index.html', 'text/html', None), ('zuul-info/index.html', 'text/html', None),
]) ])
top_index = self.find_file(fl, 'index.html') top_index = self.find_file(fl, 'index.html')
page = open(top_index.full_path).read() page = open(top_index.full_path).read()
@@ -282,141 +291,154 @@ class TestFileList(testtools.TestCase):
def test_topdir_parent_link(self): def test_topdir_parent_link(self):
'''Test index generation creates topdir parent link''' '''Test index generation creates topdir parent link'''
with FileList() as fl: with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs/')) with FileFixture() as file_fixture:
ix = Indexer(fl) fl.add(os.path.join(file_fixture.root, 'logs/'))
ix.make_indexes( ix = Indexer(fl)
create_parent_links=True, ix.make_indexes(
create_topdir_parent_link=True) create_parent_links=True,
create_topdir_parent_link=True)
self.assert_files(fl, [ self.assert_files(fl, [
('', 'application/directory', None), ('', 'application/directory', None),
('controller', 'application/directory', None), ('controller', 'application/directory', None),
('zuul-info', 'application/directory', None), ('zuul-info', 'application/directory', None),
('job-output.json', 'application/json', None), ('job-output.json', 'application/json', None),
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
'text/plain', None), 'text/plain', None),
('index.html', 'text/html', None), ('index.html', 'text/html', None),
('controller/subdir', 'application/directory', None), ('controller/subdir', 'application/directory', None),
('controller/compressed.gz', 'text/plain', 'gzip'), ('controller/compressed.gz', 'text/plain', 'gzip'),
('controller/cpu-load.svg', 'image/svg+xml', None), ('controller/cpu-load.svg', 'image/svg+xml', None),
('controller/journal.xz', 'text/plain', 'xz'), ('controller/journal.xz', 'text/plain', 'xz'),
('controller/service_log.txt', 'text/plain', None), ('controller/service_log.txt', 'text/plain', None),
('controller/syslog', 'text/plain', None), ('controller/syslog', 'text/plain', None),
('controller/index.html', 'text/html', None), ('controller/index.html', 'text/html', None),
('controller/subdir/foo::3.txt', 'text/plain', None), ('controller/subdir/foo::3.txt', 'text/plain', None),
('controller/subdir/subdir.txt', 'text/plain', None), ('controller/subdir/subdir.txt', 'text/plain', None),
('controller/subdir/index.html', 'text/html', None), ('controller/subdir/index.html', 'text/html', None),
('zuul-info/inventory.yaml', 'text/plain', None), ('zuul-info/inventory.yaml', 'text/plain', None),
('zuul-info/zuul-info.controller.txt', 'text/plain', None), ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
('zuul-info/index.html', 'text/html', None), ('zuul-info/index.html', 'text/html', None),
]) ])
top_index = self.find_file(fl, 'index.html') top_index = self.find_file(fl, 'index.html')
page = open(top_index.full_path).read() page = open(top_index.full_path).read()
page = BeautifulSoup(page, 'html.parser') page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:] rows = page.find_all('tr')[1:]
self.assertEqual(len(rows), 5) self.assertEqual(len(rows), 5)
self.assertEqual(rows[0].find('a').get('href'), self.assertEqual(rows[0].find('a').get('href'),
'../index.html') '../index.html')
self.assertEqual(rows[0].find('a').text, '../') self.assertEqual(rows[0].find('a').text, '../')
self.assertEqual(rows[1].find('a').get('href'), self.assertEqual(rows[1].find('a').get('href'),
'controller/index.html') 'controller/index.html')
self.assertEqual(rows[1].find('a').text, 'controller/') self.assertEqual(rows[1].find('a').text, 'controller/')
self.assertEqual(rows[2].find('a').get('href'), self.assertEqual(rows[2].find('a').get('href'),
'zuul-info/index.html') 'zuul-info/index.html')
self.assertEqual(rows[2].find('a').text, 'zuul-info/') self.assertEqual(rows[2].find('a').text, 'zuul-info/')
subdir_index = self.find_file(fl, 'controller/subdir/index.html') subdir_index = self.find_file(
page = open(subdir_index.full_path).read() fl, 'controller/subdir/index.html'
page = BeautifulSoup(page, 'html.parser') )
rows = page.find_all('tr')[1:] page = open(subdir_index.full_path).read()
self.assertEqual(rows[0].find('a').get('href'), '../index.html') page = BeautifulSoup(page, 'html.parser')
self.assertEqual(rows[0].find('a').text, '../') rows = page.find_all('tr')[1:]
self.assertEqual(
rows[0].find('a').get('href'), '../index.html'
)
self.assertEqual(rows[0].find('a').text, '../')
# Test proper escaping of files with funny names # Test proper escaping of files with funny names
self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt') self.assertEqual(
self.assertEqual(rows[1].find('a').text, 'foo::3.txt') rows[1].find('a').get('href'), 'foo%3A%3A3.txt'
# Test files without escaping )
self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt') self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
self.assertEqual(rows[2].find('a').text, 'subdir.txt') # Test files without escaping
self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
self.assertEqual(rows[2].find('a').text, 'subdir.txt')
def test_no_parent_links(self): def test_no_parent_links(self):
'''Test index generation creates topdir parent link''' '''Test index generation creates topdir parent link'''
with FileList() as fl: with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs/')) with FileFixture() as file_fixture:
ix = Indexer(fl) fl.add(os.path.join(file_fixture.root, 'logs/'))
ix.make_indexes( ix = Indexer(fl)
create_parent_links=False, ix.make_indexes(
create_topdir_parent_link=False) create_parent_links=False,
create_topdir_parent_link=False)
self.assert_files(fl, [ self.assert_files(fl, [
('', 'application/directory', None), ('', 'application/directory', None),
('controller', 'application/directory', None), ('controller', 'application/directory', None),
('zuul-info', 'application/directory', None), ('zuul-info', 'application/directory', None),
('job-output.json', 'application/json', None), ('job-output.json', 'application/json', None),
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
'text/plain', None), 'text/plain', None),
('index.html', 'text/html', None), ('index.html', 'text/html', None),
('controller/subdir', 'application/directory', None), ('controller/subdir', 'application/directory', None),
('controller/compressed.gz', 'text/plain', 'gzip'), ('controller/compressed.gz', 'text/plain', 'gzip'),
('controller/cpu-load.svg', 'image/svg+xml', None), ('controller/cpu-load.svg', 'image/svg+xml', None),
('controller/journal.xz', 'text/plain', 'xz'), ('controller/journal.xz', 'text/plain', 'xz'),
('controller/service_log.txt', 'text/plain', None), ('controller/service_log.txt', 'text/plain', None),
('controller/syslog', 'text/plain', None), ('controller/syslog', 'text/plain', None),
('controller/index.html', 'text/html', None), ('controller/index.html', 'text/html', None),
('controller/subdir/foo::3.txt', 'text/plain', None), ('controller/subdir/foo::3.txt', 'text/plain', None),
('controller/subdir/subdir.txt', 'text/plain', None), ('controller/subdir/subdir.txt', 'text/plain', None),
('controller/subdir/index.html', 'text/html', None), ('controller/subdir/index.html', 'text/html', None),
('zuul-info/inventory.yaml', 'text/plain', None), ('zuul-info/inventory.yaml', 'text/plain', None),
('zuul-info/zuul-info.controller.txt', 'text/plain', None), ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
('zuul-info/index.html', 'text/html', None), ('zuul-info/index.html', 'text/html', None),
]) ])
top_index = self.find_file(fl, 'index.html') top_index = self.find_file(fl, 'index.html')
page = open(top_index.full_path).read() page = open(top_index.full_path).read()
page = BeautifulSoup(page, 'html.parser') page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:] rows = page.find_all('tr')[1:]
self.assertEqual(len(rows), 4) self.assertEqual(len(rows), 4)
self.assertEqual(rows[0].find('a').get('href'), self.assertEqual(rows[0].find('a').get('href'),
'controller/index.html') 'controller/index.html')
self.assertEqual(rows[0].find('a').text, self.assertEqual(rows[0].find('a').text,
'controller/') 'controller/')
self.assertEqual(rows[1].find('a').get('href'), self.assertEqual(rows[1].find('a').get('href'),
'zuul-info/index.html') 'zuul-info/index.html')
self.assertEqual(rows[1].find('a').text, self.assertEqual(rows[1].find('a').text,
'zuul-info/') 'zuul-info/')
subdir_index = self.find_file(fl, 'controller/subdir/index.html') subdir_index = self.find_file(
page = open(subdir_index.full_path).read() fl, 'controller/subdir/index.html'
page = BeautifulSoup(page, 'html.parser') )
rows = page.find_all('tr')[1:] page = open(subdir_index.full_path).read()
page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:]
# Test proper escaping of files with funny names # Test proper escaping of files with funny names
self.assertEqual(rows[0].find('a').get('href'), 'foo%3A%3A3.txt') self.assertEqual(
self.assertEqual(rows[0].find('a').text, 'foo::3.txt') rows[0].find('a').get('href'), 'foo%3A%3A3.txt'
# Test files without escaping )
self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt') self.assertEqual(rows[0].find('a').text, 'foo::3.txt')
self.assertEqual(rows[1].find('a').text, 'subdir.txt') # Test files without escaping
self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
self.assertEqual(rows[1].find('a').text, 'subdir.txt')
class TestFileDetail(testtools.TestCase): class TestFileDetail(testtools.TestCase):
def test_get_file_detail(self): def test_get_file_detail(self):
'''Test files info''' '''Test files info'''
path = os.path.join(FIXTURE_DIR, 'logs/job-output.json') with FileFixture() as file_fixture:
file_detail = FileDetail(path, '') path = os.path.join(file_fixture.root, 'logs/job-output.json')
path_stat = os.stat(path) file_detail = FileDetail(path, '')
self.assertEqual( path_stat = os.stat(path)
time.gmtime(path_stat[stat.ST_MTIME]), self.assertEqual(
file_detail.last_modified) time.gmtime(path_stat[stat.ST_MTIME]),
self.assertEqual(16, file_detail.size) file_detail.last_modified)
self.assertEqual(16, file_detail.size)
def test_get_file_detail_missing_file(self): def test_get_file_detail_missing_file(self):
'''Test files that go missing during a walk''' '''Test files that go missing during a walk'''
@@ -456,26 +478,29 @@ class TestUpload(testtools.TestCase):
) )
# Get some test files to upload # Get some test files to upload
files = [ with FileFixture() as file_fixture:
FileDetail( files = [
os.path.join(FIXTURE_DIR, "logs/job-output.json"), FileDetail(
"job-output.json", os.path.join(file_fixture.root, "logs/job-output.json"),
), "job-output.json",
FileDetail(
os.path.join(FIXTURE_DIR, "logs/zuul-info/inventory.yaml"),
"inventory.yaml",
),
]
expected_failures = [
{
"file": "job-output.json",
"error": (
"Error posting file after multiple attempts: "
"Failed for a reason"
), ),
}, FileDetail(
] os.path.join(
file_fixture.root, "logs/zuul-info/inventory.yaml"
),
"inventory.yaml",
),
]
failures = uploader.upload(files) expected_failures = [
self.assertEqual(expected_failures, failures) {
"file": "job-output.json",
"error": (
"Error posting file after multiple attempts: "
"Failed for a reason"
),
},
]
failures = uploader.upload(files)
self.assertEqual(expected_failures, failures)