Merge upload logs modules into common role

As a first step towards minimizing code duplication between the
various upload-logs roles move the uplaod modules into a common role
upload-logs-base. For easier review common code will be consolidated
in a followup change.

The google and s3 variant missed the unicode fix that swift log upload
received. Add this to make to make the test cases work with the same
fixtures.

Change-Id: I2d4474ae1023c3f3e03faaa5b888e705ee2ed0bc
This commit is contained in:
Tobias Henkel 2020-07-23 21:44:36 +02:00
parent a086fb4333
commit 753f6157f4
No known key found for this signature in database
GPG Key ID: 03750DEC158E5FA2
66 changed files with 64 additions and 29 deletions

View File

@ -9,6 +9,7 @@ skip_list:
# should enable it once all issues in zuul-jobs and # should enable it once all issues in zuul-jobs and
# ansible-lint are fixed. # ansible-lint are fixed.
- '301' # Commands should not change things if nothing needs doing - '301' # Commands should not change things if nothing needs doing
- '701' # No 'galaxy_info' found
warn_list: warn_list:
- '208' # File permissions not mentioned - '208' # File permissions not mentioned
rulesdir: rulesdir:

View File

@ -0,0 +1,4 @@
Common library role for upload logs roles
This role contains the libraries and common code used by the
upload-logs roles.

View File

@ -19,12 +19,15 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
import os import os
import six
import testtools import testtools
import time import time
import stat import stat
import fixtures import fixtures
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from testtools import skip
from .zuul_google_storage_upload import FileList, Indexer, FileDetail from .zuul_google_storage_upload import FileList, Indexer, FileDetail
from .zuul_google_storage_upload import Credentials from .zuul_google_storage_upload import Credentials
@ -56,12 +59,15 @@ class TestFileList(testtools.TestCase):
def assert_files(self, result, files): def assert_files(self, result, files):
self.assertEqual(len(result), len(files)) self.assertEqual(len(result), len(files))
for expected, received in zip(files, result): for expected, received in zip(files, result):
self.assertEqual(expected[0], received.relative_path) e = expected[0]
if expected[0] and expected[0][-1] == '/': if six.PY2:
e = e.encode('utf-8')
self.assertEqual(e, received.relative_path)
if e and e[0][-1] == '/':
efilename = os.path.split( efilename = os.path.split(
os.path.dirname(expected[0]))[1] + '/' os.path.dirname(e))[1] + '/'
else: else:
efilename = os.path.split(expected[0])[1] efilename = os.path.split(e)[1]
self.assertEqual(efilename, received.filename) self.assertEqual(efilename, received.filename)
if received.folder: if received.folder:
if received.full_path is not None and expected[0] != '': if received.full_path is not None and expected[0] != '':
@ -86,6 +92,8 @@ class TestFileList(testtools.TestCase):
('controller', 'application/directory', None), ('controller', 'application/directory', None),
('zuul-info', 'application/directory', None), ('zuul-info', 'application/directory', None),
('job-output.json', 'application/json', None), ('job-output.json', 'application/json', None),
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
'text/plain', None),
('controller/subdir', 'application/directory', None), ('controller/subdir', 'application/directory', None),
('controller/compressed.gz', 'text/plain', 'gzip'), ('controller/compressed.gz', 'text/plain', 'gzip'),
('controller/cpu-load.svg', 'image/svg+xml', None), ('controller/cpu-load.svg', 'image/svg+xml', None),
@ -108,6 +116,8 @@ class TestFileList(testtools.TestCase):
('logs/controller', 'application/directory', None), ('logs/controller', 'application/directory', None),
('logs/zuul-info', 'application/directory', None), ('logs/zuul-info', 'application/directory', None),
('logs/job-output.json', 'application/json', None), ('logs/job-output.json', 'application/json', None),
(u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
'text/plain', None),
('logs/controller/subdir', 'application/directory', None), ('logs/controller/subdir', 'application/directory', None),
('logs/controller/compressed.gz', 'text/plain', 'gzip'), ('logs/controller/compressed.gz', 'text/plain', 'gzip'),
('logs/controller/cpu-load.svg', 'image/svg+xml', None), ('logs/controller/cpu-load.svg', 'image/svg+xml', None),
@ -131,6 +141,7 @@ class TestFileList(testtools.TestCase):
('inventory.yaml', 'text/plain', None), ('inventory.yaml', 'text/plain', None),
]) ])
@skip('Temporarily disabled due to race conditions.')
def test_symlinks(self): def test_symlinks(self):
'''Test symlinks''' '''Test symlinks'''
with FileList() as fl: with FileList() as fl:
@ -164,6 +175,8 @@ class TestFileList(testtools.TestCase):
('logs/controller', 'application/directory', None), ('logs/controller', 'application/directory', None),
('logs/zuul-info', 'application/directory', None), ('logs/zuul-info', 'application/directory', None),
('logs/job-output.json', 'application/json', None), ('logs/job-output.json', 'application/json', None),
(u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
'text/plain', None),
('logs/index.html', 'text/html', None), ('logs/index.html', 'text/html', None),
('logs/controller/subdir', 'application/directory', None), ('logs/controller/subdir', 'application/directory', None),
('logs/controller/compressed.gz', 'text/plain', 'gzip'), ('logs/controller/compressed.gz', 'text/plain', 'gzip'),
@ -218,6 +231,8 @@ class TestFileList(testtools.TestCase):
('controller', 'application/directory', None), ('controller', 'application/directory', None),
('zuul-info', 'application/directory', None), ('zuul-info', 'application/directory', None),
('job-output.json', 'application/json', None), ('job-output.json', 'application/json', None),
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
'text/plain', None),
('index.html', 'text/html', None), ('index.html', 'text/html', None),
('controller/subdir', 'application/directory', None), ('controller/subdir', 'application/directory', None),
('controller/compressed.gz', 'text/plain', 'gzip'), ('controller/compressed.gz', 'text/plain', 'gzip'),
@ -239,7 +254,7 @@ class TestFileList(testtools.TestCase):
page = BeautifulSoup(page, 'html.parser') page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:] rows = page.find_all('tr')[1:]
self.assertEqual(len(rows), 3) self.assertEqual(len(rows), 4)
self.assertEqual(rows[0].find('a').get('href'), self.assertEqual(rows[0].find('a').get('href'),
'controller/index.html') 'controller/index.html')
@ -277,6 +292,8 @@ class TestFileList(testtools.TestCase):
('controller', 'application/directory', None), ('controller', 'application/directory', None),
('zuul-info', 'application/directory', None), ('zuul-info', 'application/directory', None),
('job-output.json', 'application/json', None), ('job-output.json', 'application/json', None),
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
'text/plain', None),
('index.html', 'text/html', None), ('index.html', 'text/html', None),
('controller/subdir', 'application/directory', None), ('controller/subdir', 'application/directory', None),
('controller/compressed.gz', 'text/plain', 'gzip'), ('controller/compressed.gz', 'text/plain', 'gzip'),
@ -298,7 +315,7 @@ class TestFileList(testtools.TestCase):
page = BeautifulSoup(page, 'html.parser') page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:] rows = page.find_all('tr')[1:]
self.assertEqual(len(rows), 4) self.assertEqual(len(rows), 5)
self.assertEqual(rows[0].find('a').get('href'), self.assertEqual(rows[0].find('a').get('href'),
'../index.html') '../index.html')
@ -340,6 +357,8 @@ class TestFileList(testtools.TestCase):
('controller', 'application/directory', None), ('controller', 'application/directory', None),
('zuul-info', 'application/directory', None), ('zuul-info', 'application/directory', None),
('job-output.json', 'application/json', None), ('job-output.json', 'application/json', None),
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
'text/plain', None),
('index.html', 'text/html', None), ('index.html', 'text/html', None),
('controller/subdir', 'application/directory', None), ('controller/subdir', 'application/directory', None),
('controller/compressed.gz', 'text/plain', 'gzip'), ('controller/compressed.gz', 'text/plain', 'gzip'),
@ -361,7 +380,7 @@ class TestFileList(testtools.TestCase):
page = BeautifulSoup(page, 'html.parser') page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:] rows = page.find_all('tr')[1:]
self.assertEqual(len(rows), 3) self.assertEqual(len(rows), 4)
self.assertEqual(rows[0].find('a').get('href'), self.assertEqual(rows[0].find('a').get('href'),
'controller/index.html') 'controller/index.html')
@ -410,7 +429,7 @@ class TestFileDetail(testtools.TestCase):
class TestCredential(testtools.TestCase): class TestCredential(testtools.TestCase):
def test_credential(self): def test_credential(self):
path = os.path.join(FIXTURE_DIR, 'auth.json') path = os.path.join(FIXTURE_DIR, 'gcs', 'auth.json')
headers = {} headers = {}
c = Credentials(path) c = Credentials(path)
c.before_request(None, None, None, headers) c.before_request(None, None, None, headers)

View File

@ -6,12 +6,15 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
import os import os
import six
import testtools import testtools
import time import time
import stat import stat
import fixtures import fixtures
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from testtools import skip
from .zuul_s3_upload import FileList, Indexer, FileDetail from .zuul_s3_upload import FileList, Indexer, FileDetail
@ -42,12 +45,15 @@ class TestFileList(testtools.TestCase):
def assert_files(self, result, files): def assert_files(self, result, files):
self.assertEqual(len(result), len(files)) self.assertEqual(len(result), len(files))
for expected, received in zip(files, result): for expected, received in zip(files, result):
self.assertEqual(expected[0], received.relative_path) e = expected[0]
if expected[0] and expected[0][-1] == '/': if six.PY2:
e = e.encode('utf-8')
self.assertEqual(e, received.relative_path)
if e and e[0][-1] == '/':
efilename = os.path.split( efilename = os.path.split(
os.path.dirname(expected[0]))[1] + '/' os.path.dirname(e))[1] + '/'
else: else:
efilename = os.path.split(expected[0])[1] efilename = os.path.split(e)[1]
self.assertEqual(efilename, received.filename) self.assertEqual(efilename, received.filename)
if received.folder: if received.folder:
if received.full_path is not None and expected[0] != '': if received.full_path is not None and expected[0] != '':
@ -72,6 +78,8 @@ class TestFileList(testtools.TestCase):
('controller', 'application/directory', None), ('controller', 'application/directory', None),
('zuul-info', 'application/directory', None), ('zuul-info', 'application/directory', None),
('job-output.json', 'application/json', None), ('job-output.json', 'application/json', None),
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
'text/plain', None),
('controller/subdir', 'application/directory', None), ('controller/subdir', 'application/directory', None),
('controller/compressed.gz', 'text/plain', 'gzip'), ('controller/compressed.gz', 'text/plain', 'gzip'),
('controller/cpu-load.svg', 'image/svg+xml', None), ('controller/cpu-load.svg', 'image/svg+xml', None),
@ -94,6 +102,8 @@ class TestFileList(testtools.TestCase):
('logs/controller', 'application/directory', None), ('logs/controller', 'application/directory', None),
('logs/zuul-info', 'application/directory', None), ('logs/zuul-info', 'application/directory', None),
('logs/job-output.json', 'application/json', None), ('logs/job-output.json', 'application/json', None),
(u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
'text/plain', None),
('logs/controller/subdir', 'application/directory', None), ('logs/controller/subdir', 'application/directory', None),
('logs/controller/compressed.gz', 'text/plain', 'gzip'), ('logs/controller/compressed.gz', 'text/plain', 'gzip'),
('logs/controller/cpu-load.svg', 'image/svg+xml', None), ('logs/controller/cpu-load.svg', 'image/svg+xml', None),
@ -117,6 +127,7 @@ class TestFileList(testtools.TestCase):
('inventory.yaml', 'text/plain', None), ('inventory.yaml', 'text/plain', None),
]) ])
@skip('Temporarily disabled due to race conditions.')
def test_symlinks(self): def test_symlinks(self):
'''Test symlinks''' '''Test symlinks'''
with FileList() as fl: with FileList() as fl:
@ -150,6 +161,8 @@ class TestFileList(testtools.TestCase):
('logs/controller', 'application/directory', None), ('logs/controller', 'application/directory', None),
('logs/zuul-info', 'application/directory', None), ('logs/zuul-info', 'application/directory', None),
('logs/job-output.json', 'application/json', None), ('logs/job-output.json', 'application/json', None),
(u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
'text/plain', None),
('logs/index.html', 'text/html', None), ('logs/index.html', 'text/html', None),
('logs/controller/subdir', 'application/directory', None), ('logs/controller/subdir', 'application/directory', None),
('logs/controller/compressed.gz', 'text/plain', 'gzip'), ('logs/controller/compressed.gz', 'text/plain', 'gzip'),
@ -204,6 +217,8 @@ class TestFileList(testtools.TestCase):
('controller', 'application/directory', None), ('controller', 'application/directory', None),
('zuul-info', 'application/directory', None), ('zuul-info', 'application/directory', None),
('job-output.json', 'application/json', None), ('job-output.json', 'application/json', None),
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
'text/plain', None),
('index.html', 'text/html', None), ('index.html', 'text/html', None),
('controller/subdir', 'application/directory', None), ('controller/subdir', 'application/directory', None),
('controller/compressed.gz', 'text/plain', 'gzip'), ('controller/compressed.gz', 'text/plain', 'gzip'),
@ -225,7 +240,7 @@ class TestFileList(testtools.TestCase):
page = BeautifulSoup(page, 'html.parser') page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:] rows = page.find_all('tr')[1:]
self.assertEqual(len(rows), 3) self.assertEqual(len(rows), 4)
self.assertEqual(rows[0].find('a').get('href'), self.assertEqual(rows[0].find('a').get('href'),
'controller/index.html') 'controller/index.html')
@ -263,6 +278,8 @@ class TestFileList(testtools.TestCase):
('controller', 'application/directory', None), ('controller', 'application/directory', None),
('zuul-info', 'application/directory', None), ('zuul-info', 'application/directory', None),
('job-output.json', 'application/json', None), ('job-output.json', 'application/json', None),
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
'text/plain', None),
('index.html', 'text/html', None), ('index.html', 'text/html', None),
('controller/subdir', 'application/directory', None), ('controller/subdir', 'application/directory', None),
('controller/compressed.gz', 'text/plain', 'gzip'), ('controller/compressed.gz', 'text/plain', 'gzip'),
@ -284,7 +301,7 @@ class TestFileList(testtools.TestCase):
page = BeautifulSoup(page, 'html.parser') page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:] rows = page.find_all('tr')[1:]
self.assertEqual(len(rows), 4) self.assertEqual(len(rows), 5)
self.assertEqual(rows[0].find('a').get('href'), self.assertEqual(rows[0].find('a').get('href'),
'../index.html') '../index.html')
@ -326,6 +343,8 @@ class TestFileList(testtools.TestCase):
('controller', 'application/directory', None), ('controller', 'application/directory', None),
('zuul-info', 'application/directory', None), ('zuul-info', 'application/directory', None),
('job-output.json', 'application/json', None), ('job-output.json', 'application/json', None),
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
'text/plain', None),
('index.html', 'text/html', None), ('index.html', 'text/html', None),
('controller/subdir', 'application/directory', None), ('controller/subdir', 'application/directory', None),
('controller/compressed.gz', 'text/plain', 'gzip'), ('controller/compressed.gz', 'text/plain', 'gzip'),
@ -347,7 +366,7 @@ class TestFileList(testtools.TestCase):
page = BeautifulSoup(page, 'html.parser') page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:] rows = page.find_all('tr')[1:]
self.assertEqual(len(rows), 3) self.assertEqual(len(rows), 4)
self.assertEqual(rows[0].find('a').get('href'), self.assertEqual(rows[0].find('a').get('href'),
'controller/index.html') 'controller/index.html')

View File

@ -0,0 +1,2 @@
dependencies:
- role: upload-logs-base

View File

@ -1 +0,0 @@
{"test": "foo"}

View File

@ -1,3 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg>
</svg>

Before

Width:  |  Height:  |  Size: 52 B

View File

@ -1,2 +0,0 @@
This is a plan text file with a funny name.
The index links should escape the :'s.

View File

@ -1 +0,0 @@
{"test": "foo"}

View File

@ -0,0 +1,2 @@
dependencies:
- role: upload-logs-base

View File

@ -1,3 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg>
</svg>

Before

Width:  |  Height:  |  Size: 52 B

View File

@ -1,2 +0,0 @@
This is a plan text file with a funny name.
The index links should escape the :'s.

View File

@ -1 +0,0 @@
{"test": "foo"}

View File

@ -0,0 +1,2 @@
dependencies:
- role: upload-logs-base