Use urlencoded filenames in test fixtures
When checking out repo to Windows environment, some files in test fixture file trees do not get checked out. By setting renaming those files with an urlencoded name, and populating a temporary directory with corresponding file structure and decoded filenames during test by a fixtures script, this will not cause checkout problems for those who want to read the repo in Win environment Change-Id: If354eaf3f0bf2e86ddc05e3750aa5f0067dd9e21
This commit is contained in:
Daniel Blixt
parent
57a2f59f5a
commit
dfbc8e2ec8
1
roles/generate-zuul-manifest/library/test-fixtures/.gitattributes
vendored
Normal file
1
roles/generate-zuul-manifest/library/test-fixtures/.gitattributes
vendored
Normal file
@ -0,0 +1 @@
|
||||
*.* eol=lf
|
||||
139
roles/test-upload-logs-swift/library/filefixture.py
Normal file
139
roles/test-upload-logs-swift/library/filefixture.py
Normal file
@ -0,0 +1,139 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Handle file name special characters in a file tree.
|
||||
|
||||
All files stored in a filetree can be renamed to urlencoded filenames.
|
||||
A file tree can also be copied to a temporary location with file names
|
||||
decoded, to be used in tests with special characters that are not always
|
||||
possible to store on all file systems.
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
try:
|
||||
from urllib.parse import quote as urlib_quote
|
||||
from urllib.parse import unquote as urlib_unquote
|
||||
except ImportError:
|
||||
from urllib import quote as urlib_quote
|
||||
from urllib import unquote as urlib_unquote
|
||||
import argparse
|
||||
import fixtures
|
||||
import tempfile
|
||||
import shutil
|
||||
|
||||
|
||||
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
|
||||
'test-fixtures')
|
||||
|
||||
SAFE_CHARS = "\\/"
|
||||
|
||||
|
||||
def portable_makedirs_exist_ok(path):
|
||||
try:
|
||||
os.makedirs(path, exist_ok=True)
|
||||
except TypeError as err:
|
||||
if "unexpected keyword argument" not in str(err):
|
||||
raise err
|
||||
if not os.path.exists(path):
|
||||
try:
|
||||
os.makedirs(path)
|
||||
except OSError as err:
|
||||
if "File exists" not in err:
|
||||
raise err
|
||||
|
||||
|
||||
def urlencode_filetree():
|
||||
for root, _, files in os.walk(FIXTURE_DIR):
|
||||
for filename in files:
|
||||
os.rename(
|
||||
os.path.join(root, filename),
|
||||
os.path.join(
|
||||
root, urlib_quote(urlib_unquote(filename), SAFE_CHARS)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def populate_filetree(dst_dir=None):
|
||||
|
||||
if not os.path.exists(FIXTURE_DIR):
|
||||
return None
|
||||
|
||||
if not dst_dir:
|
||||
dst_dir = tempfile.mkdtemp()
|
||||
|
||||
portable_makedirs_exist_ok(dst_dir)
|
||||
|
||||
for root, dirs, files in os.walk(FIXTURE_DIR):
|
||||
dst_root = root.replace(FIXTURE_DIR, dst_dir, 1)
|
||||
for directory in dirs:
|
||||
portable_makedirs_exist_ok(os.path.join(dst_root, directory))
|
||||
for filename in files:
|
||||
try:
|
||||
shutil.copyfile(
|
||||
os.path.join(root, filename),
|
||||
os.path.join(dst_root, urlib_unquote(filename))
|
||||
)
|
||||
except IOError as err:
|
||||
print(
|
||||
"\nFile {}".format(
|
||||
os.path.join(dst_root, urlib_unquote(filename))
|
||||
),
|
||||
"\nnot possible to write to disk,",
|
||||
"\npossibly due to filename not being valid on Windows?\n"
|
||||
)
|
||||
shutil.rmtree(dst_dir)
|
||||
raise err
|
||||
|
||||
return dst_dir
|
||||
|
||||
|
||||
class FileFixture(fixtures.Fixture):
|
||||
|
||||
def _setUp(self):
|
||||
self.root = tempfile.mkdtemp()
|
||||
self.addCleanup(self.local_clean_up)
|
||||
populate_filetree(self.root)
|
||||
# There is no cleanup action, as the filetree is left intact for other
|
||||
# tests to use
|
||||
|
||||
def local_clean_up(self):
|
||||
shutil.rmtree(self.root)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
parser = argparse.ArgumentParser(__doc__)
|
||||
parser.add_argument(
|
||||
'--populate',
|
||||
help="Causes files in {}".format(FIXTURE_DIR) +
|
||||
"to be copied with decoded file name to a tmp dir" +
|
||||
"Overrides --encode",
|
||||
action='store_true'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--encode',
|
||||
help="Causes files under {} to be renamed with urlencoding.".format(
|
||||
FIXTURE_DIR
|
||||
) + "DEFAULT behaviour, overridden by --populate",
|
||||
action='store_true'
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.populate:
|
||||
print(populate_filetree())
|
||||
else:
|
||||
urlencode_filetree()
|
||||
1
roles/test-upload-logs-swift/library/test-fixtures/.gitattributes
vendored
Normal file
1
roles/test-upload-logs-swift/library/test-fixtures/.gitattributes
vendored
Normal file
@ -0,0 +1 @@
|
||||
*.* eol=lf
|
||||
@ -32,7 +32,7 @@ except ImportError:
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from .zuul_swift_upload import FileList, Indexer, FileDetail, Uploader
|
||||
|
||||
from .filefixture import FileFixture
|
||||
|
||||
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
|
||||
'test-fixtures')
|
||||
@ -50,8 +50,11 @@ class SymlinkFixture(fixtures.Fixture):
|
||||
]
|
||||
|
||||
def _setUp(self):
|
||||
self.file_fixture = FileFixture()
|
||||
self.file_fixture.setUp()
|
||||
self.addCleanup(self.file_fixture.cleanUp)
|
||||
for (src, target) in self.links:
|
||||
path = os.path.join(FIXTURE_DIR, 'links', src)
|
||||
path = os.path.join(self.file_fixture.root, 'links', src)
|
||||
os.symlink(target, path)
|
||||
self.addCleanup(os.unlink, path)
|
||||
|
||||
@ -88,66 +91,69 @@ class TestFileList(testtools.TestCase):
|
||||
'''Test a single directory with a trailing slash'''
|
||||
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
])
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs/'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
])
|
||||
|
||||
def test_single_dir(self):
|
||||
'''Test a single directory without a trailing slash'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('logs', 'application/directory', None),
|
||||
('logs/controller', 'application/directory', None),
|
||||
('logs/zuul-info', 'application/directory', None),
|
||||
('logs/job-output.json', 'application/json', None),
|
||||
(u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('logs/controller/subdir', 'application/directory', None),
|
||||
('logs/controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('logs/controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('logs/controller/journal.xz', 'text/plain', 'xz'),
|
||||
('logs/controller/service_log.txt', 'text/plain', None),
|
||||
('logs/controller/syslog', 'text/plain', None),
|
||||
('logs/controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('logs/controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('logs/zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('logs/zuul-info/zuul-info.controller.txt',
|
||||
'text/plain', None),
|
||||
])
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('logs', 'application/directory', None),
|
||||
('logs/controller', 'application/directory', None),
|
||||
('logs/zuul-info', 'application/directory', None),
|
||||
('logs/job-output.json', 'application/json', None),
|
||||
(u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('logs/controller/subdir', 'application/directory', None),
|
||||
('logs/controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('logs/controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('logs/controller/journal.xz', 'text/plain', 'xz'),
|
||||
('logs/controller/service_log.txt', 'text/plain', None),
|
||||
('logs/controller/syslog', 'text/plain', None),
|
||||
('logs/controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('logs/controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('logs/zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('logs/zuul-info/zuul-info.controller.txt',
|
||||
'text/plain', None),
|
||||
])
|
||||
|
||||
def test_single_file(self):
|
||||
'''Test a single file'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR,
|
||||
'logs/zuul-info/inventory.yaml'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('inventory.yaml', 'text/plain', None),
|
||||
])
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root,
|
||||
'logs/zuul-info/inventory.yaml'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('inventory.yaml', 'text/plain', None),
|
||||
])
|
||||
|
||||
def test_symlinks(self):
|
||||
'''Test symlinks'''
|
||||
with FileList() as fl:
|
||||
self.useFixture(SymlinkFixture())
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'links/'))
|
||||
symlink_fixture = self.useFixture(SymlinkFixture())
|
||||
fl.add(os.path.join(symlink_fixture.file_fixture.root, 'links/'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
@ -165,7 +171,8 @@ class TestFileList(testtools.TestCase):
|
||||
def test_index_files(self):
|
||||
'''Test index generation'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs'))
|
||||
symlink_fixture = self.useFixture(SymlinkFixture())
|
||||
fl.add(os.path.join(symlink_fixture.file_fixture.root, 'logs'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes()
|
||||
|
||||
@ -223,32 +230,33 @@ class TestFileList(testtools.TestCase):
|
||||
def test_index_files_trailing_slash(self):
|
||||
'''Test index generation with a trailing slash'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes()
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes()
|
||||
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
@ -280,134 +288,145 @@ class TestFileList(testtools.TestCase):
|
||||
def test_topdir_parent_link(self):
|
||||
'''Test index generation creates topdir parent link'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes(
|
||||
create_parent_links=True,
|
||||
create_topdir_parent_link=True)
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes(
|
||||
create_parent_links=True,
|
||||
create_topdir_parent_link=True)
|
||||
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
|
||||
self.assertEqual(len(rows), 5)
|
||||
self.assertEqual(len(rows), 5)
|
||||
|
||||
self.assertEqual(rows[0].find('a').get('href'), '../')
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
self.assertEqual(rows[0].find('a').get('href'), '../')
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'controller/')
|
||||
self.assertEqual(rows[1].find('a').text, 'controller/')
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'controller/')
|
||||
self.assertEqual(rows[1].find('a').text, 'controller/')
|
||||
|
||||
self.assertEqual(rows[2].find('a').get('href'), 'zuul-info/')
|
||||
self.assertEqual(rows[2].find('a').text, 'zuul-info/')
|
||||
self.assertEqual(rows[2].find('a').get('href'), 'zuul-info/')
|
||||
self.assertEqual(rows[2].find('a').text, 'zuul-info/')
|
||||
|
||||
subdir_index = self.find_file(fl, 'controller/subdir/index.html')
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
self.assertEqual(rows[0].find('a').get('href'), '../')
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
subdir_index = self.find_file(
|
||||
fl, 'controller/subdir/index.html'
|
||||
)
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
self.assertEqual(rows[0].find('a').get('href'), '../')
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt')
|
||||
self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[2].find('a').text, 'subdir.txt')
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(
|
||||
rows[1].find('a').get('href'), 'foo%3A%3A3.txt'
|
||||
)
|
||||
self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[2].find('a').text, 'subdir.txt')
|
||||
|
||||
def test_no_parent_links(self):
|
||||
'''Test index generation creates topdir parent link'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes(
|
||||
create_parent_links=False,
|
||||
create_topdir_parent_link=False)
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes(
|
||||
create_parent_links=False,
|
||||
create_topdir_parent_link=False)
|
||||
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
|
||||
self.assertEqual(len(rows), 4)
|
||||
self.assertEqual(len(rows), 4)
|
||||
|
||||
self.assertEqual(rows[0].find('a').get('href'), 'controller/')
|
||||
self.assertEqual(rows[0].find('a').text, 'controller/')
|
||||
self.assertEqual(rows[0].find('a').get('href'), 'controller/')
|
||||
self.assertEqual(rows[0].find('a').text, 'controller/')
|
||||
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
|
||||
self.assertEqual(rows[1].find('a').text, 'zuul-info/')
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
|
||||
self.assertEqual(rows[1].find('a').text, 'zuul-info/')
|
||||
|
||||
subdir_index = self.find_file(fl, 'controller/subdir/index.html')
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
subdir_index = self.find_file(
|
||||
fl, 'controller/subdir/index.html'
|
||||
)
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(rows[0].find('a').get('href'), 'foo%3A%3A3.txt')
|
||||
self.assertEqual(rows[0].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[1].find('a').text, 'subdir.txt')
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(
|
||||
rows[0].find('a').get('href'), 'foo%3A%3A3.txt'
|
||||
)
|
||||
self.assertEqual(rows[0].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[1].find('a').text, 'subdir.txt')
|
||||
|
||||
|
||||
class TestFileDetail(testtools.TestCase):
|
||||
|
||||
def test_get_file_detail(self):
|
||||
'''Test files info'''
|
||||
path = os.path.join(FIXTURE_DIR, 'logs/job-output.json')
|
||||
file_detail = FileDetail(path, '')
|
||||
path_stat = os.stat(path)
|
||||
self.assertEqual(
|
||||
time.gmtime(path_stat[stat.ST_MTIME]),
|
||||
file_detail.last_modified)
|
||||
self.assertEqual(16, file_detail.size)
|
||||
with FileFixture() as file_fixture:
|
||||
path = os.path.join(file_fixture.root, 'logs/job-output.json')
|
||||
file_detail = FileDetail(path, '')
|
||||
path_stat = os.stat(path)
|
||||
self.assertEqual(
|
||||
time.gmtime(path_stat[stat.ST_MTIME]),
|
||||
file_detail.last_modified)
|
||||
self.assertEqual(16, file_detail.size)
|
||||
|
||||
def test_get_file_detail_missing_file(self):
|
||||
'''Test files that go missing during a walk'''
|
||||
@ -447,26 +466,29 @@ class TestUpload(testtools.TestCase):
|
||||
)
|
||||
|
||||
# Get some test files to upload
|
||||
files = [
|
||||
FileDetail(
|
||||
os.path.join(FIXTURE_DIR, "logs/job-output.json"),
|
||||
"job-output.json",
|
||||
),
|
||||
FileDetail(
|
||||
os.path.join(FIXTURE_DIR, "logs/zuul-info/inventory.yaml"),
|
||||
"inventory.yaml",
|
||||
),
|
||||
]
|
||||
|
||||
expected_failures = [
|
||||
{
|
||||
"file": "job-output.json",
|
||||
"error": (
|
||||
"Error posting file after multiple attempts: "
|
||||
"Failed for a reason"
|
||||
with FileFixture() as file_fixture:
|
||||
files = [
|
||||
FileDetail(
|
||||
os.path.join(file_fixture.root, "logs/job-output.json"),
|
||||
"job-output.json",
|
||||
),
|
||||
},
|
||||
]
|
||||
FileDetail(
|
||||
os.path.join(
|
||||
file_fixture.root, "logs/zuul-info/inventory.yaml"
|
||||
),
|
||||
"inventory.yaml",
|
||||
),
|
||||
]
|
||||
|
||||
failures = uploader.upload(files)
|
||||
self.assertEqual(expected_failures, failures)
|
||||
expected_failures = [
|
||||
{
|
||||
"file": "job-output.json",
|
||||
"error": (
|
||||
"Error posting file after multiple attempts: "
|
||||
"Failed for a reason"
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
failures = uploader.upload(files)
|
||||
self.assertEqual(expected_failures, failures)
|
||||
|
||||
139
roles/upload-logs-base/library/filefixture.py
Normal file
139
roles/upload-logs-base/library/filefixture.py
Normal file
@ -0,0 +1,139 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Handle file name special characters in a file tree.
|
||||
|
||||
All files stored in a filetree can be renamed to urlencoded filenames.
|
||||
A file tree can also be copied to a temporary location with file names
|
||||
decoded, to be used in tests with special characters that are not always
|
||||
possible to store on all file systems.
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
try:
|
||||
from urllib.parse import quote as urlib_quote
|
||||
from urllib.parse import unquote as urlib_unquote
|
||||
except ImportError:
|
||||
from urllib import quote as urlib_quote
|
||||
from urllib import unquote as urlib_unquote
|
||||
import argparse
|
||||
import fixtures
|
||||
import tempfile
|
||||
import shutil
|
||||
|
||||
|
||||
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
|
||||
'test-fixtures')
|
||||
|
||||
SAFE_CHARS = "\\/"
|
||||
|
||||
|
||||
def portable_makedirs_exist_ok(path):
|
||||
try:
|
||||
os.makedirs(path, exist_ok=True)
|
||||
except TypeError as err:
|
||||
if "unexpected keyword argument" not in str(err):
|
||||
raise err
|
||||
if not os.path.exists(path):
|
||||
try:
|
||||
os.makedirs(path)
|
||||
except OSError as err:
|
||||
if "File exists" not in err:
|
||||
raise err
|
||||
|
||||
|
||||
def urlencode_filetree():
|
||||
for root, _, files in os.walk(FIXTURE_DIR):
|
||||
for filename in files:
|
||||
os.rename(
|
||||
os.path.join(root, filename),
|
||||
os.path.join(
|
||||
root, urlib_quote(urlib_unquote(filename), SAFE_CHARS)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def populate_filetree(dst_dir=None):
|
||||
|
||||
if not os.path.exists(FIXTURE_DIR):
|
||||
return None
|
||||
|
||||
if not dst_dir:
|
||||
dst_dir = tempfile.mkdtemp()
|
||||
|
||||
portable_makedirs_exist_ok(dst_dir)
|
||||
|
||||
for root, dirs, files in os.walk(FIXTURE_DIR):
|
||||
dst_root = root.replace(FIXTURE_DIR, dst_dir, 1)
|
||||
for directory in dirs:
|
||||
portable_makedirs_exist_ok(os.path.join(dst_root, directory))
|
||||
for filename in files:
|
||||
try:
|
||||
shutil.copyfile(
|
||||
os.path.join(root, filename),
|
||||
os.path.join(dst_root, urlib_unquote(filename))
|
||||
)
|
||||
except IOError as err:
|
||||
print(
|
||||
"\nFile {}".format(
|
||||
os.path.join(dst_root, urlib_unquote(filename))
|
||||
),
|
||||
"\nnot possible to write to disk,",
|
||||
"\npossibly due to filename not being valid on Windows?\n"
|
||||
)
|
||||
shutil.rmtree(dst_dir)
|
||||
raise err
|
||||
|
||||
return dst_dir
|
||||
|
||||
|
||||
class FileFixture(fixtures.Fixture):
|
||||
|
||||
def _setUp(self):
|
||||
self.root = tempfile.mkdtemp()
|
||||
self.addCleanup(self.local_clean_up)
|
||||
populate_filetree(self.root)
|
||||
# There is no cleanup action, as the filetree is left intact for other
|
||||
# tests to use
|
||||
|
||||
def local_clean_up(self):
|
||||
shutil.rmtree(self.root)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
parser = argparse.ArgumentParser(__doc__)
|
||||
parser.add_argument(
|
||||
'--populate',
|
||||
help="Causes files in {}".format(FIXTURE_DIR) +
|
||||
"to be copied with decoded file name to a tmp dir" +
|
||||
"Overrides --encode",
|
||||
action='store_true'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--encode',
|
||||
help="Causes files under {} to be renamed with urlencoding.".format(
|
||||
FIXTURE_DIR
|
||||
) + "DEFAULT behaviour, overridden by --populate",
|
||||
action='store_true'
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.populate:
|
||||
print(populate_filetree())
|
||||
else:
|
||||
urlencode_filetree()
|
||||
1
roles/upload-logs-base/library/test-fixtures/.gitattributes
vendored
Normal file
1
roles/upload-logs-base/library/test-fixtures/.gitattributes
vendored
Normal file
@ -0,0 +1 @@
|
||||
*.* eol=lf
|
||||
@ -33,6 +33,7 @@ import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from .zuul_swift_upload import Uploader
|
||||
from ..module_utils.zuul_jobs.upload_utils import FileList, Indexer, FileDetail
|
||||
from .filefixture import FileFixture
|
||||
|
||||
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
|
||||
'test-fixtures')
|
||||
@ -50,8 +51,11 @@ class SymlinkFixture(fixtures.Fixture):
|
||||
]
|
||||
|
||||
def _setUp(self):
|
||||
self.file_fixture = FileFixture()
|
||||
self.file_fixture.setUp()
|
||||
self.addCleanup(self.file_fixture.cleanUp)
|
||||
for (src, target) in self.links:
|
||||
path = os.path.join(FIXTURE_DIR, 'links', src)
|
||||
path = os.path.join(self.file_fixture.root, 'links', src)
|
||||
os.symlink(target, path)
|
||||
self.addCleanup(os.unlink, path)
|
||||
|
||||
@ -88,66 +92,69 @@ class TestFileList(testtools.TestCase):
|
||||
'''Test a single directory with a trailing slash'''
|
||||
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
])
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs/'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
])
|
||||
|
||||
def test_single_dir(self):
|
||||
'''Test a single directory without a trailing slash'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('logs', 'application/directory', None),
|
||||
('logs/controller', 'application/directory', None),
|
||||
('logs/zuul-info', 'application/directory', None),
|
||||
('logs/job-output.json', 'application/json', None),
|
||||
(u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('logs/controller/subdir', 'application/directory', None),
|
||||
('logs/controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('logs/controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('logs/controller/journal.xz', 'text/plain', 'xz'),
|
||||
('logs/controller/service_log.txt', 'text/plain', None),
|
||||
('logs/controller/syslog', 'text/plain', None),
|
||||
('logs/controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('logs/controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('logs/zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('logs/zuul-info/zuul-info.controller.txt',
|
||||
'text/plain', None),
|
||||
])
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('logs', 'application/directory', None),
|
||||
('logs/controller', 'application/directory', None),
|
||||
('logs/zuul-info', 'application/directory', None),
|
||||
('logs/job-output.json', 'application/json', None),
|
||||
(u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('logs/controller/subdir', 'application/directory', None),
|
||||
('logs/controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('logs/controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('logs/controller/journal.xz', 'text/plain', 'xz'),
|
||||
('logs/controller/service_log.txt', 'text/plain', None),
|
||||
('logs/controller/syslog', 'text/plain', None),
|
||||
('logs/controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('logs/controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('logs/zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('logs/zuul-info/zuul-info.controller.txt',
|
||||
'text/plain', None),
|
||||
])
|
||||
|
||||
def test_single_file(self):
|
||||
'''Test a single file'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR,
|
||||
'logs/zuul-info/inventory.yaml'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('inventory.yaml', 'text/plain', None),
|
||||
])
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root,
|
||||
'logs/zuul-info/inventory.yaml'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('inventory.yaml', 'text/plain', None),
|
||||
])
|
||||
|
||||
def test_symlinks(self):
|
||||
'''Test symlinks'''
|
||||
with FileList() as fl:
|
||||
self.useFixture(SymlinkFixture())
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'links/'))
|
||||
symlink_fixture = self.useFixture(SymlinkFixture())
|
||||
fl.add(os.path.join(symlink_fixture.file_fixture.root, 'links/'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
@ -165,7 +172,8 @@ class TestFileList(testtools.TestCase):
|
||||
def test_index_files(self):
|
||||
'''Test index generation'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs'))
|
||||
symlink_fixture = self.useFixture(SymlinkFixture())
|
||||
fl.add(os.path.join(symlink_fixture.file_fixture.root, 'logs'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes()
|
||||
|
||||
@ -223,32 +231,33 @@ class TestFileList(testtools.TestCase):
|
||||
def test_index_files_trailing_slash(self):
|
||||
'''Test index generation with a trailing slash'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes()
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes()
|
||||
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
@ -282,141 +291,154 @@ class TestFileList(testtools.TestCase):
|
||||
def test_topdir_parent_link(self):
|
||||
'''Test index generation creates topdir parent link'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes(
|
||||
create_parent_links=True,
|
||||
create_topdir_parent_link=True)
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes(
|
||||
create_parent_links=True,
|
||||
create_topdir_parent_link=True)
|
||||
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
|
||||
self.assertEqual(len(rows), 5)
|
||||
self.assertEqual(len(rows), 5)
|
||||
|
||||
self.assertEqual(rows[0].find('a').get('href'),
|
||||
'../index.html')
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
self.assertEqual(rows[0].find('a').get('href'),
|
||||
'../index.html')
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
|
||||
self.assertEqual(rows[1].find('a').get('href'),
|
||||
'controller/index.html')
|
||||
self.assertEqual(rows[1].find('a').text, 'controller/')
|
||||
self.assertEqual(rows[1].find('a').get('href'),
|
||||
'controller/index.html')
|
||||
self.assertEqual(rows[1].find('a').text, 'controller/')
|
||||
|
||||
self.assertEqual(rows[2].find('a').get('href'),
|
||||
'zuul-info/index.html')
|
||||
self.assertEqual(rows[2].find('a').text, 'zuul-info/')
|
||||
self.assertEqual(rows[2].find('a').get('href'),
|
||||
'zuul-info/index.html')
|
||||
self.assertEqual(rows[2].find('a').text, 'zuul-info/')
|
||||
|
||||
subdir_index = self.find_file(fl, 'controller/subdir/index.html')
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
self.assertEqual(rows[0].find('a').get('href'), '../index.html')
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
subdir_index = self.find_file(
|
||||
fl, 'controller/subdir/index.html'
|
||||
)
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
self.assertEqual(
|
||||
rows[0].find('a').get('href'), '../index.html'
|
||||
)
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt')
|
||||
self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[2].find('a').text, 'subdir.txt')
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(
|
||||
rows[1].find('a').get('href'), 'foo%3A%3A3.txt'
|
||||
)
|
||||
self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[2].find('a').text, 'subdir.txt')
|
||||
|
||||
def test_no_parent_links(self):
|
||||
'''Test index generation creates topdir parent link'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes(
|
||||
create_parent_links=False,
|
||||
create_topdir_parent_link=False)
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes(
|
||||
create_parent_links=False,
|
||||
create_topdir_parent_link=False)
|
||||
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
|
||||
self.assertEqual(len(rows), 4)
|
||||
self.assertEqual(len(rows), 4)
|
||||
|
||||
self.assertEqual(rows[0].find('a').get('href'),
|
||||
'controller/index.html')
|
||||
self.assertEqual(rows[0].find('a').text,
|
||||
'controller/')
|
||||
self.assertEqual(rows[0].find('a').get('href'),
|
||||
'controller/index.html')
|
||||
self.assertEqual(rows[0].find('a').text,
|
||||
'controller/')
|
||||
|
||||
self.assertEqual(rows[1].find('a').get('href'),
|
||||
'zuul-info/index.html')
|
||||
self.assertEqual(rows[1].find('a').text,
|
||||
'zuul-info/')
|
||||
self.assertEqual(rows[1].find('a').get('href'),
|
||||
'zuul-info/index.html')
|
||||
self.assertEqual(rows[1].find('a').text,
|
||||
'zuul-info/')
|
||||
|
||||
subdir_index = self.find_file(fl, 'controller/subdir/index.html')
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
subdir_index = self.find_file(
|
||||
fl, 'controller/subdir/index.html'
|
||||
)
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(rows[0].find('a').get('href'), 'foo%3A%3A3.txt')
|
||||
self.assertEqual(rows[0].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[1].find('a').text, 'subdir.txt')
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(
|
||||
rows[0].find('a').get('href'), 'foo%3A%3A3.txt'
|
||||
)
|
||||
self.assertEqual(rows[0].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[1].find('a').text, 'subdir.txt')
|
||||
|
||||
|
||||
class TestFileDetail(testtools.TestCase):
|
||||
|
||||
def test_get_file_detail(self):
|
||||
'''Test files info'''
|
||||
path = os.path.join(FIXTURE_DIR, 'logs/job-output.json')
|
||||
file_detail = FileDetail(path, '')
|
||||
path_stat = os.stat(path)
|
||||
self.assertEqual(
|
||||
time.gmtime(path_stat[stat.ST_MTIME]),
|
||||
file_detail.last_modified)
|
||||
self.assertEqual(16, file_detail.size)
|
||||
with FileFixture() as file_fixture:
|
||||
path = os.path.join(file_fixture.root, 'logs/job-output.json')
|
||||
file_detail = FileDetail(path, '')
|
||||
path_stat = os.stat(path)
|
||||
self.assertEqual(
|
||||
time.gmtime(path_stat[stat.ST_MTIME]),
|
||||
file_detail.last_modified)
|
||||
self.assertEqual(16, file_detail.size)
|
||||
|
||||
def test_get_file_detail_missing_file(self):
|
||||
'''Test files that go missing during a walk'''
|
||||
@ -456,26 +478,29 @@ class TestUpload(testtools.TestCase):
|
||||
)
|
||||
|
||||
# Get some test files to upload
|
||||
files = [
|
||||
FileDetail(
|
||||
os.path.join(FIXTURE_DIR, "logs/job-output.json"),
|
||||
"job-output.json",
|
||||
),
|
||||
FileDetail(
|
||||
os.path.join(FIXTURE_DIR, "logs/zuul-info/inventory.yaml"),
|
||||
"inventory.yaml",
|
||||
),
|
||||
]
|
||||
|
||||
expected_failures = [
|
||||
{
|
||||
"file": "job-output.json",
|
||||
"error": (
|
||||
"Error posting file after multiple attempts: "
|
||||
"Failed for a reason"
|
||||
with FileFixture() as file_fixture:
|
||||
files = [
|
||||
FileDetail(
|
||||
os.path.join(file_fixture.root, "logs/job-output.json"),
|
||||
"job-output.json",
|
||||
),
|
||||
},
|
||||
]
|
||||
FileDetail(
|
||||
os.path.join(
|
||||
file_fixture.root, "logs/zuul-info/inventory.yaml"
|
||||
),
|
||||
"inventory.yaml",
|
||||
),
|
||||
]
|
||||
|
||||
failures = uploader.upload(files)
|
||||
self.assertEqual(expected_failures, failures)
|
||||
expected_failures = [
|
||||
{
|
||||
"file": "job-output.json",
|
||||
"error": (
|
||||
"Error posting file after multiple attempts: "
|
||||
"Failed for a reason"
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
failures = uploader.upload(files)
|
||||
self.assertEqual(expected_failures, failures)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user