zuul/zuul-jobs
Code Issues Proposed changes

Merge "Use urlencoded filenames in test fixtures"

Browse Source
This commit is contained in:
Zuul
2021-02-01 14:11:44 +00:00
committed by Gerrit Code Review
parent 79c44ead3a dfbc8e2ec8
commit 5bb91e867b
11 changed files with 734 additions and 406 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
roles/generate-zuul-manifest/library/test-fixtures/.gitattributes vendored Normal file
View File

@@ -0,0 +1 @@
*.* eol=lf

139
roles/test-upload-logs-swift/library/filefixture.py Normal file
View File

@@ -0,0 +1,139 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Handle file name special characters in a file tree.
All files stored in a filetree can be renamed to urlencoded filenames.
A file tree can also be copied to a temporary location with file names
decoded, to be used in tests with special characters that are not always
possible to store on all file systems.
"""
from __future__ import print_function
import os
try:
from urllib.parse import quote as urlib_quote
from urllib.parse import unquote as urlib_unquote
except ImportError:
from urllib import quote as urlib_quote
from urllib import unquote as urlib_unquote
import argparse
import fixtures
import tempfile
import shutil
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
'test-fixtures')
SAFE_CHARS = "\\/"
def portable_makedirs_exist_ok(path):
try:
os.makedirs(path, exist_ok=True)
except TypeError as err:
if "unexpected keyword argument" not in str(err):
raise err
if not os.path.exists(path):
try:
os.makedirs(path)
except OSError as err:
if "File exists" not in err:
raise err
def urlencode_filetree():
for root, _, files in os.walk(FIXTURE_DIR):
for filename in files:
os.rename(
os.path.join(root, filename),
os.path.join(
root, urlib_quote(urlib_unquote(filename), SAFE_CHARS)
)
)
def populate_filetree(dst_dir=None):
if not os.path.exists(FIXTURE_DIR):
return None
if not dst_dir:
dst_dir = tempfile.mkdtemp()
portable_makedirs_exist_ok(dst_dir)
for root, dirs, files in os.walk(FIXTURE_DIR):
dst_root = root.replace(FIXTURE_DIR, dst_dir, 1)
for directory in dirs:
portable_makedirs_exist_ok(os.path.join(dst_root, directory))
for filename in files:
try:
shutil.copyfile(
os.path.join(root, filename),
os.path.join(dst_root, urlib_unquote(filename))
)
except IOError as err:
print(
"\nFile {}".format(
os.path.join(dst_root, urlib_unquote(filename))
),
"\nnot possible to write to disk,",
"\npossibly due to filename not being valid on Windows?\n"
)
shutil.rmtree(dst_dir)
raise err
return dst_dir
class FileFixture(fixtures.Fixture):
def _setUp(self):
self.root = tempfile.mkdtemp()
self.addCleanup(self.local_clean_up)
populate_filetree(self.root)
# There is no cleanup action, as the filetree is left intact for other
# tests to use
def local_clean_up(self):
shutil.rmtree(self.root)
if __name__ == '__main__':
parser = argparse.ArgumentParser(__doc__)
parser.add_argument(
'--populate',
help="Causes files in {}".format(FIXTURE_DIR) +
"to be copied with decoded file name to a tmp dir" +
"Overrides --encode",
action='store_true'
)
parser.add_argument(
'--encode',
help="Causes files under {} to be renamed with urlencoding.".format(
FIXTURE_DIR
) + "DEFAULT behaviour, overridden by --populate",
action='store_true'
)
args = parser.parse_args()
if args.populate:
print(populate_filetree())
else:
urlencode_filetree()

1
roles/test-upload-logs-swift/library/test-fixtures/.gitattributes vendored Normal file
View File

@@ -0,0 +1 @@
*.* eol=lf

0
roles/test-upload-logs-swift/library/test-fixtures/logs/Ꮓບບξ-unicode.txt → roles/test-upload-logs-swift/library/test-fixtures/logs/%E1%8F%83%E0%BA%9A%E0%BA%9A%CE%BE-unicode.txt
View File

0
roles/test-upload-logs-swift/library/test-fixtures/logs/controller/subdir/foo::3.txt → roles/test-upload-logs-swift/library/test-fixtures/logs/controller/subdir/foo%3A%3A3.txt
View File

58
roles/test-upload-logs-swift/library/test_zuul_swift_upload.py
View File

@@ -32,7 +32,7 @@ except ImportError:
import requests
from bs4 import BeautifulSoup
from .zuul_swift_upload import FileList, Indexer, FileDetail, Uploader
from .filefixture import FileFixture
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
'test-fixtures')
@@ -50,8 +50,11 @@ class SymlinkFixture(fixtures.Fixture):
]
def _setUp(self):
self.file_fixture = FileFixture()
self.file_fixture.setUp()
self.addCleanup(self.file_fixture.cleanUp)
for (src, target) in self.links:
path = os.path.join(FIXTURE_DIR, 'links', src)
path = os.path.join(self.file_fixture.root, 'links', src)
os.symlink(target, path)
self.addCleanup(os.unlink, path)
@@ -88,7 +91,8 @@ class TestFileList(testtools.TestCase):
'''Test a single directory with a trailing slash'''
with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
with FileFixture() as file_fixture:
fl.add(os.path.join(file_fixture.root, 'logs/'))
self.assert_files(fl, [
('', 'application/directory', None),
('controller', 'application/directory', None),
@@ -111,7 +115,8 @@ class TestFileList(testtools.TestCase):
def test_single_dir(self):
'''Test a single directory without a trailing slash'''
with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs'))
with FileFixture() as file_fixture:
fl.add(os.path.join(file_fixture.root, 'logs'))
self.assert_files(fl, [
('', 'application/directory', None),
('logs', 'application/directory', None),
@@ -136,7 +141,8 @@ class TestFileList(testtools.TestCase):
def test_single_file(self):
'''Test a single file'''
with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR,
with FileFixture() as file_fixture:
fl.add(os.path.join(file_fixture.root,
'logs/zuul-info/inventory.yaml'))
self.assert_files(fl, [
('', 'application/directory', None),
@@ -146,8 +152,8 @@ class TestFileList(testtools.TestCase):
def test_symlinks(self):
'''Test symlinks'''
with FileList() as fl:
self.useFixture(SymlinkFixture())
fl.add(os.path.join(FIXTURE_DIR, 'links/'))
symlink_fixture = self.useFixture(SymlinkFixture())
fl.add(os.path.join(symlink_fixture.file_fixture.root, 'links/'))
self.assert_files(fl, [
('', 'application/directory', None),
('controller', 'application/directory', None),
@@ -165,7 +171,8 @@ class TestFileList(testtools.TestCase):
def test_index_files(self):
'''Test index generation'''
with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs'))
symlink_fixture = self.useFixture(SymlinkFixture())
fl.add(os.path.join(symlink_fixture.file_fixture.root, 'logs'))
ix = Indexer(fl)
ix.make_indexes()
@@ -223,7 +230,8 @@ class TestFileList(testtools.TestCase):
def test_index_files_trailing_slash(self):
'''Test index generation with a trailing slash'''
with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
with FileFixture() as file_fixture:
fl.add(os.path.join(file_fixture.root, 'logs/'))
ix = Indexer(fl)
ix.make_indexes()
@@ -280,7 +288,8 @@ class TestFileList(testtools.TestCase):
def test_topdir_parent_link(self):
'''Test index generation creates topdir parent link'''
with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
with FileFixture() as file_fixture:
fl.add(os.path.join(file_fixture.root, 'logs/'))
ix = Indexer(fl)
ix.make_indexes(
create_parent_links=True,
@@ -325,7 +334,9 @@ class TestFileList(testtools.TestCase):
self.assertEqual(rows[2].find('a').get('href'), 'zuul-info/')
self.assertEqual(rows[2].find('a').text, 'zuul-info/')
subdir_index = self.find_file(fl, 'controller/subdir/index.html')
subdir_index = self.find_file(
fl, 'controller/subdir/index.html'
)
page = open(subdir_index.full_path).read()
page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:]
@@ -333,7 +344,9 @@ class TestFileList(testtools.TestCase):
self.assertEqual(rows[0].find('a').text, '../')
# Test proper escaping of files with funny names
self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt')
self.assertEqual(
rows[1].find('a').get('href'), 'foo%3A%3A3.txt'
)
self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
# Test files without escaping
self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
@@ -342,7 +355,8 @@ class TestFileList(testtools.TestCase):
def test_no_parent_links(self):
'''Test index generation creates topdir parent link'''
with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
with FileFixture() as file_fixture:
fl.add(os.path.join(file_fixture.root, 'logs/'))
ix = Indexer(fl)
ix.make_indexes(
create_parent_links=False,
@@ -384,13 +398,17 @@ class TestFileList(testtools.TestCase):
self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
self.assertEqual(rows[1].find('a').text, 'zuul-info/')
subdir_index = self.find_file(fl, 'controller/subdir/index.html')
subdir_index = self.find_file(
fl, 'controller/subdir/index.html'
)
page = open(subdir_index.full_path).read()
page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:]
# Test proper escaping of files with funny names
self.assertEqual(rows[0].find('a').get('href'), 'foo%3A%3A3.txt')
self.assertEqual(
rows[0].find('a').get('href'), 'foo%3A%3A3.txt'
)
self.assertEqual(rows[0].find('a').text, 'foo::3.txt')
# Test files without escaping
self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
@@ -401,7 +419,8 @@ class TestFileDetail(testtools.TestCase):
def test_get_file_detail(self):
'''Test files info'''
path = os.path.join(FIXTURE_DIR, 'logs/job-output.json')
with FileFixture() as file_fixture:
path = os.path.join(file_fixture.root, 'logs/job-output.json')
file_detail = FileDetail(path, '')
path_stat = os.stat(path)
self.assertEqual(
@@ -447,13 +466,16 @@ class TestUpload(testtools.TestCase):
)
# Get some test files to upload
with FileFixture() as file_fixture:
files = [
FileDetail(
os.path.join(FIXTURE_DIR, "logs/job-output.json"),
os.path.join(file_fixture.root, "logs/job-output.json"),
"job-output.json",
),
FileDetail(
os.path.join(FIXTURE_DIR, "logs/zuul-info/inventory.yaml"),
os.path.join(
file_fixture.root, "logs/zuul-info/inventory.yaml"
),
"inventory.yaml",
),
]

139
roles/upload-logs-base/library/filefixture.py Normal file
View File

@@ -0,0 +1,139 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Handle file name special characters in a file tree.
All files stored in a filetree can be renamed to urlencoded filenames.
A file tree can also be copied to a temporary location with file names
decoded, to be used in tests with special characters that are not always
possible to store on all file systems.
"""
from __future__ import print_function
import os
try:
from urllib.parse import quote as urlib_quote
from urllib.parse import unquote as urlib_unquote
except ImportError:
from urllib import quote as urlib_quote
from urllib import unquote as urlib_unquote
import argparse
import fixtures
import tempfile
import shutil
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
'test-fixtures')
SAFE_CHARS = "\\/"
def portable_makedirs_exist_ok(path):
try:
os.makedirs(path, exist_ok=True)
except TypeError as err:
if "unexpected keyword argument" not in str(err):
raise err
if not os.path.exists(path):
try:
os.makedirs(path)
except OSError as err:
if "File exists" not in err:
raise err
def urlencode_filetree():
for root, _, files in os.walk(FIXTURE_DIR):
for filename in files:
os.rename(
os.path.join(root, filename),
os.path.join(
root, urlib_quote(urlib_unquote(filename), SAFE_CHARS)
)
)
def populate_filetree(dst_dir=None):
if not os.path.exists(FIXTURE_DIR):
return None
if not dst_dir:
dst_dir = tempfile.mkdtemp()
portable_makedirs_exist_ok(dst_dir)
for root, dirs, files in os.walk(FIXTURE_DIR):
dst_root = root.replace(FIXTURE_DIR, dst_dir, 1)
for directory in dirs:
portable_makedirs_exist_ok(os.path.join(dst_root, directory))
for filename in files:
try:
shutil.copyfile(
os.path.join(root, filename),
os.path.join(dst_root, urlib_unquote(filename))
)
except IOError as err:
print(
"\nFile {}".format(
os.path.join(dst_root, urlib_unquote(filename))
),
"\nnot possible to write to disk,",
"\npossibly due to filename not being valid on Windows?\n"
)
shutil.rmtree(dst_dir)
raise err
return dst_dir
class FileFixture(fixtures.Fixture):
def _setUp(self):
self.root = tempfile.mkdtemp()
self.addCleanup(self.local_clean_up)
populate_filetree(self.root)
# There is no cleanup action, as the filetree is left intact for other
# tests to use
def local_clean_up(self):
shutil.rmtree(self.root)
if __name__ == '__main__':
parser = argparse.ArgumentParser(__doc__)
parser.add_argument(
'--populate',
help="Causes files in {}".format(FIXTURE_DIR) +
"to be copied with decoded file name to a tmp dir" +
"Overrides --encode",
action='store_true'
)
parser.add_argument(
'--encode',
help="Causes files under {} to be renamed with urlencoding.".format(
FIXTURE_DIR
) + "DEFAULT behaviour, overridden by --populate",
action='store_true'
)
args = parser.parse_args()
if args.populate:
print(populate_filetree())
else:
urlencode_filetree()

1
roles/upload-logs-base/library/test-fixtures/.gitattributes vendored Normal file
View File

@@ -0,0 +1 @@
*.* eol=lf

0
roles/upload-logs-base/library/test-fixtures/logs/Ꮓບບξ-unicode.txt → roles/upload-logs-base/library/test-fixtures/logs/%E1%8F%83%E0%BA%9A%E0%BA%9A%CE%BE-unicode.txt
View File

0
roles/upload-logs-base/library/test-fixtures/logs/controller/subdir/foo::3.txt → roles/upload-logs-base/library/test-fixtures/logs/controller/subdir/foo%3A%3A3.txt
View File

61
roles/upload-logs-base/library/test_index.py
View File

@@ -33,6 +33,7 @@ import requests
from bs4 import BeautifulSoup
from .zuul_swift_upload import Uploader
from ..module_utils.zuul_jobs.upload_utils import FileList, Indexer, FileDetail
from .filefixture import FileFixture
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
'test-fixtures')
@@ -50,8 +51,11 @@ class SymlinkFixture(fixtures.Fixture):
]
def _setUp(self):
self.file_fixture = FileFixture()
self.file_fixture.setUp()
self.addCleanup(self.file_fixture.cleanUp)
for (src, target) in self.links:
path = os.path.join(FIXTURE_DIR, 'links', src)
path = os.path.join(self.file_fixture.root, 'links', src)
os.symlink(target, path)
self.addCleanup(os.unlink, path)
@@ -88,7 +92,8 @@ class TestFileList(testtools.TestCase):
'''Test a single directory with a trailing slash'''
with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
with FileFixture() as file_fixture:
fl.add(os.path.join(file_fixture.root, 'logs/'))
self.assert_files(fl, [
('', 'application/directory', None),
('controller', 'application/directory', None),
@@ -111,7 +116,8 @@ class TestFileList(testtools.TestCase):
def test_single_dir(self):
'''Test a single directory without a trailing slash'''
with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs'))
with FileFixture() as file_fixture:
fl.add(os.path.join(file_fixture.root, 'logs'))
self.assert_files(fl, [
('', 'application/directory', None),
('logs', 'application/directory', None),
@@ -136,7 +142,8 @@ class TestFileList(testtools.TestCase):
def test_single_file(self):
'''Test a single file'''
with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR,
with FileFixture() as file_fixture:
fl.add(os.path.join(file_fixture.root,
'logs/zuul-info/inventory.yaml'))
self.assert_files(fl, [
('', 'application/directory', None),
@@ -146,8 +153,8 @@ class TestFileList(testtools.TestCase):
def test_symlinks(self):
'''Test symlinks'''
with FileList() as fl:
self.useFixture(SymlinkFixture())
fl.add(os.path.join(FIXTURE_DIR, 'links/'))
symlink_fixture = self.useFixture(SymlinkFixture())
fl.add(os.path.join(symlink_fixture.file_fixture.root, 'links/'))
self.assert_files(fl, [
('', 'application/directory', None),
('controller', 'application/directory', None),
@@ -165,7 +172,8 @@ class TestFileList(testtools.TestCase):
def test_index_files(self):
'''Test index generation'''
with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs'))
symlink_fixture = self.useFixture(SymlinkFixture())
fl.add(os.path.join(symlink_fixture.file_fixture.root, 'logs'))
ix = Indexer(fl)
ix.make_indexes()
@@ -223,7 +231,8 @@ class TestFileList(testtools.TestCase):
def test_index_files_trailing_slash(self):
'''Test index generation with a trailing slash'''
with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
with FileFixture() as file_fixture:
fl.add(os.path.join(file_fixture.root, 'logs/'))
ix = Indexer(fl)
ix.make_indexes()
@@ -282,7 +291,8 @@ class TestFileList(testtools.TestCase):
def test_topdir_parent_link(self):
'''Test index generation creates topdir parent link'''
with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
with FileFixture() as file_fixture:
fl.add(os.path.join(file_fixture.root, 'logs/'))
ix = Indexer(fl)
ix.make_indexes(
create_parent_links=True,
@@ -330,15 +340,21 @@ class TestFileList(testtools.TestCase):
'zuul-info/index.html')
self.assertEqual(rows[2].find('a').text, 'zuul-info/')
subdir_index = self.find_file(fl, 'controller/subdir/index.html')
subdir_index = self.find_file(
fl, 'controller/subdir/index.html'
)
page = open(subdir_index.full_path).read()
page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:]
self.assertEqual(rows[0].find('a').get('href'), '../index.html')
self.assertEqual(
rows[0].find('a').get('href'), '../index.html'
)
self.assertEqual(rows[0].find('a').text, '../')
# Test proper escaping of files with funny names
self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt')
self.assertEqual(
rows[1].find('a').get('href'), 'foo%3A%3A3.txt'
)
self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
# Test files without escaping
self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
@@ -347,7 +363,8 @@ class TestFileList(testtools.TestCase):
def test_no_parent_links(self):
'''Test index generation creates topdir parent link'''
with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
with FileFixture() as file_fixture:
fl.add(os.path.join(file_fixture.root, 'logs/'))
ix = Indexer(fl)
ix.make_indexes(
create_parent_links=False,
@@ -393,13 +410,17 @@ class TestFileList(testtools.TestCase):
self.assertEqual(rows[1].find('a').text,
'zuul-info/')
subdir_index = self.find_file(fl, 'controller/subdir/index.html')
subdir_index = self.find_file(
fl, 'controller/subdir/index.html'
)
page = open(subdir_index.full_path).read()
page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:]
# Test proper escaping of files with funny names
self.assertEqual(rows[0].find('a').get('href'), 'foo%3A%3A3.txt')
self.assertEqual(
rows[0].find('a').get('href'), 'foo%3A%3A3.txt'
)
self.assertEqual(rows[0].find('a').text, 'foo::3.txt')
# Test files without escaping
self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
@@ -410,7 +431,8 @@ class TestFileDetail(testtools.TestCase):
def test_get_file_detail(self):
'''Test files info'''
path = os.path.join(FIXTURE_DIR, 'logs/job-output.json')
with FileFixture() as file_fixture:
path = os.path.join(file_fixture.root, 'logs/job-output.json')
file_detail = FileDetail(path, '')
path_stat = os.stat(path)
self.assertEqual(
@@ -456,13 +478,16 @@ class TestUpload(testtools.TestCase):
)
# Get some test files to upload
with FileFixture() as file_fixture:
files = [
FileDetail(
os.path.join(FIXTURE_DIR, "logs/job-output.json"),
os.path.join(file_fixture.root, "logs/job-output.json"),
"job-output.json",
),
FileDetail(
os.path.join(FIXTURE_DIR, "logs/zuul-info/inventory.yaml"),
os.path.join(
file_fixture.root, "logs/zuul-info/inventory.yaml"
),
"inventory.yaml",
),
]