Browse Source

Merge "upload-logs-swift: Turn FileList into a context manager"

Zuul 5 months ago
parent
commit
92c5348052

+ 252
- 248
roles/upload-logs-swift/library/test_zuul_swift_upload.py View File

@@ -82,272 +82,276 @@ class TestFileList(testtools.TestCase):
82 82
 
83 83
     def test_single_dir_trailing_slash(self):
84 84
         '''Test a single directory with a trailing slash'''
85
-        fl = FileList()
86
-        fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
87
-        self.assert_files(fl, [
88
-            ('', 'application/directory', None),
89
-            ('controller', 'application/directory', None),
90
-            ('zuul-info', 'application/directory', None),
91
-            ('job-output.json', 'application/json', None),
92
-            ('controller/subdir', 'application/directory', None),
93
-            ('controller/compressed.gz', 'text/plain', 'gzip'),
94
-            ('controller/cpu-load.svg', 'image/svg+xml', None),
95
-            ('controller/journal.xz', 'text/plain', 'xz'),
96
-            ('controller/service_log.txt', 'text/plain', None),
97
-            ('controller/syslog', 'text/plain', None),
98
-            ('controller/subdir/subdir.txt', 'text/plain', None),
99
-            ('zuul-info/inventory.yaml', 'text/plain', None),
100
-            ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
101
-        ])
85
+
86
+        with FileList() as fl:
87
+            fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
88
+            self.assert_files(fl, [
89
+                ('', 'application/directory', None),
90
+                ('controller', 'application/directory', None),
91
+                ('zuul-info', 'application/directory', None),
92
+                ('job-output.json', 'application/json', None),
93
+                ('controller/subdir', 'application/directory', None),
94
+                ('controller/compressed.gz', 'text/plain', 'gzip'),
95
+                ('controller/cpu-load.svg', 'image/svg+xml', None),
96
+                ('controller/journal.xz', 'text/plain', 'xz'),
97
+                ('controller/service_log.txt', 'text/plain', None),
98
+                ('controller/syslog', 'text/plain', None),
99
+                ('controller/subdir/subdir.txt', 'text/plain', None),
100
+                ('zuul-info/inventory.yaml', 'text/plain', None),
101
+                ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
102
+            ])
102 103
 
103 104
     def test_single_dir(self):
104 105
         '''Test a single directory without a trailing slash'''
105
-        fl = FileList()
106
-        fl.add(os.path.join(FIXTURE_DIR, 'logs'))
107
-        self.assert_files(fl, [
108
-            ('', 'application/directory', None),
109
-            ('logs', 'application/directory', None),
110
-            ('logs/controller', 'application/directory', None),
111
-            ('logs/zuul-info', 'application/directory', None),
112
-            ('logs/job-output.json', 'application/json', None),
113
-            ('logs/controller/subdir', 'application/directory', None),
114
-            ('logs/controller/compressed.gz', 'text/plain', 'gzip'),
115
-            ('logs/controller/cpu-load.svg', 'image/svg+xml', None),
116
-            ('logs/controller/journal.xz', 'text/plain', 'xz'),
117
-            ('logs/controller/service_log.txt', 'text/plain', None),
118
-            ('logs/controller/syslog', 'text/plain', None),
119
-            ('logs/controller/subdir/subdir.txt', 'text/plain', None),
120
-            ('logs/zuul-info/inventory.yaml', 'text/plain', None),
121
-            ('logs/zuul-info/zuul-info.controller.txt', 'text/plain', None),
122
-        ])
106
+        with FileList() as fl:
107
+            fl.add(os.path.join(FIXTURE_DIR, 'logs'))
108
+            self.assert_files(fl, [
109
+                ('', 'application/directory', None),
110
+                ('logs', 'application/directory', None),
111
+                ('logs/controller', 'application/directory', None),
112
+                ('logs/zuul-info', 'application/directory', None),
113
+                ('logs/job-output.json', 'application/json', None),
114
+                ('logs/controller/subdir', 'application/directory', None),
115
+                ('logs/controller/compressed.gz', 'text/plain', 'gzip'),
116
+                ('logs/controller/cpu-load.svg', 'image/svg+xml', None),
117
+                ('logs/controller/journal.xz', 'text/plain', 'xz'),
118
+                ('logs/controller/service_log.txt', 'text/plain', None),
119
+                ('logs/controller/syslog', 'text/plain', None),
120
+                ('logs/controller/subdir/subdir.txt', 'text/plain', None),
121
+                ('logs/zuul-info/inventory.yaml', 'text/plain', None),
122
+                ('logs/zuul-info/zuul-info.controller.txt',
123
+                 'text/plain', None),
124
+            ])
123 125
 
124 126
     def test_single_file(self):
125 127
         '''Test a single file'''
126
-        fl = FileList()
127
-        fl.add(os.path.join(FIXTURE_DIR,
128
-                            'logs/zuul-info/inventory.yaml'))
129
-        self.assert_files(fl, [
130
-            ('', 'application/directory', None),
131
-            ('inventory.yaml', 'text/plain', None),
132
-        ])
128
+        with FileList() as fl:
129
+            fl.add(os.path.join(FIXTURE_DIR,
130
+                                'logs/zuul-info/inventory.yaml'))
131
+            self.assert_files(fl, [
132
+                ('', 'application/directory', None),
133
+                ('inventory.yaml', 'text/plain', None),
134
+            ])
133 135
 
134 136
     def test_symlinks(self):
135 137
         '''Test symlinks'''
136
-        fl = FileList()
137
-        self.useFixture(SymlinkFixture())
138
-        fl.add(os.path.join(FIXTURE_DIR, 'links/'))
139
-        self.assert_files(fl, [
140
-            ('', 'application/directory', None),
141
-            ('controller', 'application/directory', None),
142
-            ('good_symlink', 'application/directory', None),
143
-            ('recursive_symlink', 'application/directory', None),
144
-            ('symlink_loop', 'application/directory', None),
145
-            ('symlink_loop_a', 'application/directory', None),
146
-            ('job-output.json', 'application/json', None),
147
-            ('symlink_file', 'text/plain', None),
148
-            ('controller/service_log.txt', 'text/plain', None),
149
-            ('symlink_loop/symlink_loop_b', 'application/directory', None),
150
-            ('symlink_loop/placeholder', 'text/plain', None),
151
-        ])
138
+        with FileList() as fl:
139
+            self.useFixture(SymlinkFixture())
140
+            fl.add(os.path.join(FIXTURE_DIR, 'links/'))
141
+            self.assert_files(fl, [
142
+                ('', 'application/directory', None),
143
+                ('controller', 'application/directory', None),
144
+                ('good_symlink', 'application/directory', None),
145
+                ('recursive_symlink', 'application/directory', None),
146
+                ('symlink_loop', 'application/directory', None),
147
+                ('symlink_loop_a', 'application/directory', None),
148
+                ('job-output.json', 'application/json', None),
149
+                ('symlink_file', 'text/plain', None),
150
+                ('controller/service_log.txt', 'text/plain', None),
151
+                ('symlink_loop/symlink_loop_b', 'application/directory', None),
152
+                ('symlink_loop/placeholder', 'text/plain', None),
153
+            ])
152 154
 
153 155
     def test_index_files(self):
154 156
         '''Test index generation'''
155
-        fl = FileList()
156
-        fl.add(os.path.join(FIXTURE_DIR, 'logs'))
157
-        ix = Indexer()
158
-        fl = ix.make_indexes(fl)
159
-
160
-        self.assert_files(fl, [
161
-            ('', 'application/directory', None),
162
-            ('index.html', 'text/html', None),
163
-            ('logs', 'application/directory', None),
164
-            ('logs/controller', 'application/directory', None),
165
-            ('logs/zuul-info', 'application/directory', None),
166
-            ('logs/job-output.json', 'application/json', None),
167
-            ('logs/index.html', 'text/html', None),
168
-            ('logs/controller/subdir', 'application/directory', None),
169
-            ('logs/controller/compressed.gz', 'text/plain', 'gzip'),
170
-            ('logs/controller/cpu-load.svg', 'image/svg+xml', None),
171
-            ('logs/controller/journal.xz', 'text/plain', 'xz'),
172
-            ('logs/controller/service_log.txt', 'text/plain', None),
173
-            ('logs/controller/syslog', 'text/plain', None),
174
-            ('logs/controller/index.html', 'text/html', None),
175
-            ('logs/controller/subdir/subdir.txt', 'text/plain', None),
176
-            ('logs/controller/subdir/index.html', 'text/html', None),
177
-            ('logs/zuul-info/inventory.yaml', 'text/plain', None),
178
-            ('logs/zuul-info/zuul-info.controller.txt', 'text/plain', None),
179
-            ('logs/zuul-info/index.html', 'text/html', None),
180
-        ])
181
-
182
-        top_index = self.find_file(fl, 'index.html')
183
-        page = open(top_index.full_path).read()
184
-        page = BeautifulSoup(page, 'html.parser')
185
-        rows = page.find_all('tr')[1:]
186
-
187
-        self.assertEqual(len(rows), 1)
188
-
189
-        self.assertEqual(rows[0].find('a').get('href'), 'logs/')
190
-        self.assertEqual(rows[0].find('a').text, 'logs/')
191
-
192
-        subdir_index = self.find_file(fl, 'logs/controller/subdir/index.html')
193
-        page = open(subdir_index.full_path).read()
194
-        page = BeautifulSoup(page, 'html.parser')
195
-        rows = page.find_all('tr')[1:]
196
-        self.assertEqual(rows[0].find('a').get('href'), '../')
197
-        self.assertEqual(rows[0].find('a').text, '../')
198
-
199
-        self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
200
-        self.assertEqual(rows[1].find('a').text, 'subdir.txt')
157
+        with FileList() as fl:
158
+            fl.add(os.path.join(FIXTURE_DIR, 'logs'))
159
+            ix = Indexer()
160
+            ix.make_indexes(fl)
161
+
162
+            self.assert_files(fl, [
163
+                ('', 'application/directory', None),
164
+                ('index.html', 'text/html', None),
165
+                ('logs', 'application/directory', None),
166
+                ('logs/controller', 'application/directory', None),
167
+                ('logs/zuul-info', 'application/directory', None),
168
+                ('logs/job-output.json', 'application/json', None),
169
+                ('logs/index.html', 'text/html', None),
170
+                ('logs/controller/subdir', 'application/directory', None),
171
+                ('logs/controller/compressed.gz', 'text/plain', 'gzip'),
172
+                ('logs/controller/cpu-load.svg', 'image/svg+xml', None),
173
+                ('logs/controller/journal.xz', 'text/plain', 'xz'),
174
+                ('logs/controller/service_log.txt', 'text/plain', None),
175
+                ('logs/controller/syslog', 'text/plain', None),
176
+                ('logs/controller/index.html', 'text/html', None),
177
+                ('logs/controller/subdir/subdir.txt', 'text/plain', None),
178
+                ('logs/controller/subdir/index.html', 'text/html', None),
179
+                ('logs/zuul-info/inventory.yaml', 'text/plain', None),
180
+                ('logs/zuul-info/zuul-info.controller.txt',
181
+                 'text/plain', None),
182
+                ('logs/zuul-info/index.html', 'text/html', None),
183
+            ])
184
+
185
+            top_index = self.find_file(fl, 'index.html')
186
+            page = open(top_index.full_path).read()
187
+            page = BeautifulSoup(page, 'html.parser')
188
+            rows = page.find_all('tr')[1:]
189
+
190
+            self.assertEqual(len(rows), 1)
191
+
192
+            self.assertEqual(rows[0].find('a').get('href'), 'logs/')
193
+            self.assertEqual(rows[0].find('a').text, 'logs/')
194
+
195
+            subdir_index = self.find_file(
196
+                fl, 'logs/controller/subdir/index.html')
197
+            page = open(subdir_index.full_path).read()
198
+            page = BeautifulSoup(page, 'html.parser')
199
+            rows = page.find_all('tr')[1:]
200
+            self.assertEqual(rows[0].find('a').get('href'), '../')
201
+            self.assertEqual(rows[0].find('a').text, '../')
202
+
203
+            self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
204
+            self.assertEqual(rows[1].find('a').text, 'subdir.txt')
201 205
 
202 206
     def test_index_files_trailing_slash(self):
203 207
         '''Test index generation with a trailing slash'''
204
-        fl = FileList()
205
-        fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
206
-        ix = Indexer()
207
-        fl = ix.make_indexes(fl)
208
-
209
-        self.assert_files(fl, [
210
-            ('', 'application/directory', None),
211
-            ('controller', 'application/directory', None),
212
-            ('zuul-info', 'application/directory', None),
213
-            ('job-output.json', 'application/json', None),
214
-            ('index.html', 'text/html', None),
215
-            ('controller/subdir', 'application/directory', None),
216
-            ('controller/compressed.gz', 'text/plain', 'gzip'),
217
-            ('controller/cpu-load.svg', 'image/svg+xml', None),
218
-            ('controller/journal.xz', 'text/plain', 'xz'),
219
-            ('controller/service_log.txt', 'text/plain', None),
220
-            ('controller/syslog', 'text/plain', None),
221
-            ('controller/index.html', 'text/html', None),
222
-            ('controller/subdir/subdir.txt', 'text/plain', None),
223
-            ('controller/subdir/index.html', 'text/html', None),
224
-            ('zuul-info/inventory.yaml', 'text/plain', None),
225
-            ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
226
-            ('zuul-info/index.html', 'text/html', None),
227
-        ])
228
-
229
-        top_index = self.find_file(fl, 'index.html')
230
-        page = open(top_index.full_path).read()
231
-        page = BeautifulSoup(page, 'html.parser')
232
-        rows = page.find_all('tr')[1:]
233
-
234
-        self.assertEqual(len(rows), 3)
235
-
236
-        self.assertEqual(rows[0].find('a').get('href'), 'controller/')
237
-        self.assertEqual(rows[0].find('a').text, 'controller/')
238
-
239
-        self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
240
-        self.assertEqual(rows[1].find('a').text, 'zuul-info/')
241
-
242
-        subdir_index = self.find_file(fl, 'controller/subdir/index.html')
243
-        page = open(subdir_index.full_path).read()
244
-        page = BeautifulSoup(page, 'html.parser')
245
-        rows = page.find_all('tr')[1:]
246
-        self.assertEqual(rows[0].find('a').get('href'), '../')
247
-        self.assertEqual(rows[0].find('a').text, '../')
248
-
249
-        self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
250
-        self.assertEqual(rows[1].find('a').text, 'subdir.txt')
208
+        with FileList() as fl:
209
+            fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
210
+            ix = Indexer()
211
+            ix.make_indexes(fl)
212
+
213
+            self.assert_files(fl, [
214
+                ('', 'application/directory', None),
215
+                ('controller', 'application/directory', None),
216
+                ('zuul-info', 'application/directory', None),
217
+                ('job-output.json', 'application/json', None),
218
+                ('index.html', 'text/html', None),
219
+                ('controller/subdir', 'application/directory', None),
220
+                ('controller/compressed.gz', 'text/plain', 'gzip'),
221
+                ('controller/cpu-load.svg', 'image/svg+xml', None),
222
+                ('controller/journal.xz', 'text/plain', 'xz'),
223
+                ('controller/service_log.txt', 'text/plain', None),
224
+                ('controller/syslog', 'text/plain', None),
225
+                ('controller/index.html', 'text/html', None),
226
+                ('controller/subdir/subdir.txt', 'text/plain', None),
227
+                ('controller/subdir/index.html', 'text/html', None),
228
+                ('zuul-info/inventory.yaml', 'text/plain', None),
229
+                ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
230
+                ('zuul-info/index.html', 'text/html', None),
231
+            ])
232
+
233
+            top_index = self.find_file(fl, 'index.html')
234
+            page = open(top_index.full_path).read()
235
+            page = BeautifulSoup(page, 'html.parser')
236
+            rows = page.find_all('tr')[1:]
237
+
238
+            self.assertEqual(len(rows), 3)
239
+
240
+            self.assertEqual(rows[0].find('a').get('href'), 'controller/')
241
+            self.assertEqual(rows[0].find('a').text, 'controller/')
242
+
243
+            self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
244
+            self.assertEqual(rows[1].find('a').text, 'zuul-info/')
245
+
246
+            subdir_index = self.find_file(fl, 'controller/subdir/index.html')
247
+            page = open(subdir_index.full_path).read()
248
+            page = BeautifulSoup(page, 'html.parser')
249
+            rows = page.find_all('tr')[1:]
250
+            self.assertEqual(rows[0].find('a').get('href'), '../')
251
+            self.assertEqual(rows[0].find('a').text, '../')
252
+
253
+            self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
254
+            self.assertEqual(rows[1].find('a').text, 'subdir.txt')
251 255
 
252 256
     def test_topdir_parent_link(self):
253 257
         '''Test index generation creates topdir parent link'''
254
-        fl = FileList()
255
-        fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
256
-        ix = Indexer(create_parent_links=True,
257
-                     create_topdir_parent_link=True)
258
-        fl = ix.make_indexes(fl)
259
-
260
-        self.assert_files(fl, [
261
-            ('', 'application/directory', None),
262
-            ('controller', 'application/directory', None),
263
-            ('zuul-info', 'application/directory', None),
264
-            ('job-output.json', 'application/json', None),
265
-            ('index.html', 'text/html', None),
266
-            ('controller/subdir', 'application/directory', None),
267
-            ('controller/compressed.gz', 'text/plain', 'gzip'),
268
-            ('controller/cpu-load.svg', 'image/svg+xml', None),
269
-            ('controller/journal.xz', 'text/plain', 'xz'),
270
-            ('controller/service_log.txt', 'text/plain', None),
271
-            ('controller/syslog', 'text/plain', None),
272
-            ('controller/index.html', 'text/html', None),
273
-            ('controller/subdir/subdir.txt', 'text/plain', None),
274
-            ('controller/subdir/index.html', 'text/html', None),
275
-            ('zuul-info/inventory.yaml', 'text/plain', None),
276
-            ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
277
-            ('zuul-info/index.html', 'text/html', None),
278
-        ])
279
-
280
-        top_index = self.find_file(fl, 'index.html')
281
-        page = open(top_index.full_path).read()
282
-        page = BeautifulSoup(page, 'html.parser')
283
-        rows = page.find_all('tr')[1:]
284
-
285
-        self.assertEqual(len(rows), 4)
286
-
287
-        self.assertEqual(rows[0].find('a').get('href'), '../')
288
-        self.assertEqual(rows[0].find('a').text, '../')
289
-
290
-        self.assertEqual(rows[1].find('a').get('href'), 'controller/')
291
-        self.assertEqual(rows[1].find('a').text, 'controller/')
292
-
293
-        self.assertEqual(rows[2].find('a').get('href'), 'zuul-info/')
294
-        self.assertEqual(rows[2].find('a').text, 'zuul-info/')
295
-
296
-        subdir_index = self.find_file(fl, 'controller/subdir/index.html')
297
-        page = open(subdir_index.full_path).read()
298
-        page = BeautifulSoup(page, 'html.parser')
299
-        rows = page.find_all('tr')[1:]
300
-        self.assertEqual(rows[0].find('a').get('href'), '../')
301
-        self.assertEqual(rows[0].find('a').text, '../')
302
-
303
-        self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
304
-        self.assertEqual(rows[1].find('a').text, 'subdir.txt')
258
+        with FileList() as fl:
259
+            fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
260
+            ix = Indexer(create_parent_links=True,
261
+                         create_topdir_parent_link=True)
262
+            ix.make_indexes(fl)
263
+
264
+            self.assert_files(fl, [
265
+                ('', 'application/directory', None),
266
+                ('controller', 'application/directory', None),
267
+                ('zuul-info', 'application/directory', None),
268
+                ('job-output.json', 'application/json', None),
269
+                ('index.html', 'text/html', None),
270
+                ('controller/subdir', 'application/directory', None),
271
+                ('controller/compressed.gz', 'text/plain', 'gzip'),
272
+                ('controller/cpu-load.svg', 'image/svg+xml', None),
273
+                ('controller/journal.xz', 'text/plain', 'xz'),
274
+                ('controller/service_log.txt', 'text/plain', None),
275
+                ('controller/syslog', 'text/plain', None),
276
+                ('controller/index.html', 'text/html', None),
277
+                ('controller/subdir/subdir.txt', 'text/plain', None),
278
+                ('controller/subdir/index.html', 'text/html', None),
279
+                ('zuul-info/inventory.yaml', 'text/plain', None),
280
+                ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
281
+                ('zuul-info/index.html', 'text/html', None),
282
+            ])
283
+
284
+            top_index = self.find_file(fl, 'index.html')
285
+            page = open(top_index.full_path).read()
286
+            page = BeautifulSoup(page, 'html.parser')
287
+            rows = page.find_all('tr')[1:]
288
+
289
+            self.assertEqual(len(rows), 4)
290
+
291
+            self.assertEqual(rows[0].find('a').get('href'), '../')
292
+            self.assertEqual(rows[0].find('a').text, '../')
293
+
294
+            self.assertEqual(rows[1].find('a').get('href'), 'controller/')
295
+            self.assertEqual(rows[1].find('a').text, 'controller/')
296
+
297
+            self.assertEqual(rows[2].find('a').get('href'), 'zuul-info/')
298
+            self.assertEqual(rows[2].find('a').text, 'zuul-info/')
299
+
300
+            subdir_index = self.find_file(fl, 'controller/subdir/index.html')
301
+            page = open(subdir_index.full_path).read()
302
+            page = BeautifulSoup(page, 'html.parser')
303
+            rows = page.find_all('tr')[1:]
304
+            self.assertEqual(rows[0].find('a').get('href'), '../')
305
+            self.assertEqual(rows[0].find('a').text, '../')
306
+
307
+            self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
308
+            self.assertEqual(rows[1].find('a').text, 'subdir.txt')
305 309
 
306 310
     def test_no_parent_links(self):
307 311
         '''Test index generation creates topdir parent link'''
308
-        fl = FileList()
309
-        fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
310
-        ix = Indexer(create_parent_links=False,
311
-                     create_topdir_parent_link=False)
312
-        fl = ix.make_indexes(fl)
313
-
314
-        self.assert_files(fl, [
315
-            ('', 'application/directory', None),
316
-            ('controller', 'application/directory', None),
317
-            ('zuul-info', 'application/directory', None),
318
-            ('job-output.json', 'application/json', None),
319
-            ('index.html', 'text/html', None),
320
-            ('controller/subdir', 'application/directory', None),
321
-            ('controller/compressed.gz', 'text/plain', 'gzip'),
322
-            ('controller/cpu-load.svg', 'image/svg+xml', None),
323
-            ('controller/journal.xz', 'text/plain', 'xz'),
324
-            ('controller/service_log.txt', 'text/plain', None),
325
-            ('controller/syslog', 'text/plain', None),
326
-            ('controller/index.html', 'text/html', None),
327
-            ('controller/subdir/subdir.txt', 'text/plain', None),
328
-            ('controller/subdir/index.html', 'text/html', None),
329
-            ('zuul-info/inventory.yaml', 'text/plain', None),
330
-            ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
331
-            ('zuul-info/index.html', 'text/html', None),
332
-        ])
333
-
334
-        top_index = self.find_file(fl, 'index.html')
335
-        page = open(top_index.full_path).read()
336
-        page = BeautifulSoup(page, 'html.parser')
337
-        rows = page.find_all('tr')[1:]
338
-
339
-        self.assertEqual(len(rows), 3)
340
-
341
-        self.assertEqual(rows[0].find('a').get('href'), 'controller/')
342
-        self.assertEqual(rows[0].find('a').text, 'controller/')
343
-
344
-        self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
345
-        self.assertEqual(rows[1].find('a').text, 'zuul-info/')
346
-
347
-        subdir_index = self.find_file(fl, 'controller/subdir/index.html')
348
-        page = open(subdir_index.full_path).read()
349
-        page = BeautifulSoup(page, 'html.parser')
350
-        rows = page.find_all('tr')[1:]
351
-
352
-        self.assertEqual(rows[0].find('a').get('href'), 'subdir.txt')
353
-        self.assertEqual(rows[0].find('a').text, 'subdir.txt')
312
+        with FileList() as fl:
313
+            fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
314
+            ix = Indexer(create_parent_links=False,
315
+                         create_topdir_parent_link=False)
316
+            ix.make_indexes(fl)
317
+
318
+            self.assert_files(fl, [
319
+                ('', 'application/directory', None),
320
+                ('controller', 'application/directory', None),
321
+                ('zuul-info', 'application/directory', None),
322
+                ('job-output.json', 'application/json', None),
323
+                ('index.html', 'text/html', None),
324
+                ('controller/subdir', 'application/directory', None),
325
+                ('controller/compressed.gz', 'text/plain', 'gzip'),
326
+                ('controller/cpu-load.svg', 'image/svg+xml', None),
327
+                ('controller/journal.xz', 'text/plain', 'xz'),
328
+                ('controller/service_log.txt', 'text/plain', None),
329
+                ('controller/syslog', 'text/plain', None),
330
+                ('controller/index.html', 'text/html', None),
331
+                ('controller/subdir/subdir.txt', 'text/plain', None),
332
+                ('controller/subdir/index.html', 'text/html', None),
333
+                ('zuul-info/inventory.yaml', 'text/plain', None),
334
+                ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
335
+                ('zuul-info/index.html', 'text/html', None),
336
+            ])
337
+
338
+            top_index = self.find_file(fl, 'index.html')
339
+            page = open(top_index.full_path).read()
340
+            page = BeautifulSoup(page, 'html.parser')
341
+            rows = page.find_all('tr')[1:]
342
+
343
+            self.assertEqual(len(rows), 3)
344
+
345
+            self.assertEqual(rows[0].find('a').get('href'), 'controller/')
346
+            self.assertEqual(rows[0].find('a').text, 'controller/')
347
+
348
+            self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
349
+            self.assertEqual(rows[1].find('a').text, 'zuul-info/')
350
+
351
+            subdir_index = self.find_file(fl, 'controller/subdir/index.html')
352
+            page = open(subdir_index.full_path).read()
353
+            page = BeautifulSoup(page, 'html.parser')
354
+            rows = page.find_all('tr')[1:]
355
+
356
+            self.assertEqual(rows[0].find('a').get('href'), 'subdir.txt')
357
+            self.assertEqual(rows[0].find('a').text, 'subdir.txt')

+ 53
- 35
roles/upload-logs-swift/library/zuul_swift_upload.py View File

@@ -185,10 +185,21 @@ class FileDetail():
185 185
 
186 186
 
187 187
 class FileList(Sequence):
188
+    '''A collection of FileDetail objects
189
+
190
+    This is a list-like group of FileDetail objects, intended to be
191
+    used as a context manager around the upload process.
192
+    '''
188 193
     def __init__(self):
189 194
         self.file_list = []
190 195
         self.file_list.append(FileDetail(None, '', ''))
191 196
 
197
+    def __enter__(self):
198
+        return self
199
+
200
+    def __exit__(self, type, value, traceback):
201
+        pass
202
+
192 203
     def __getitem__(self, item):
193 204
         return self.file_list.__getitem__(item)
194 205
 
@@ -269,6 +280,16 @@ class Indexer():
269 280
         self.index_filename = 'index.html'
270 281
 
271 282
     def make_indexes(self, file_list):
283
+        '''Make index files
284
+
285
+        Args:
286
+            file_list (FileList): A FileList object to be updated
287
+             with index files for each directory.
288
+        Return:
289
+            No value, the file_list will be updated
290
+        '''
291
+        assert isinstance(file_list, FileList)
292
+
272 293
         folders = collections.OrderedDict()
273 294
         for f in file_list:
274 295
             if f.folder:
@@ -302,8 +323,7 @@ class Indexer():
302 323
 
303 324
         # This appends the index file at the end of the group of files
304 325
         # for each directory.
305
-        ret_file_list = FileList()
306
-        newlist = []
326
+        new_list = []
307 327
         last_dirname = None
308 328
         for f in reversed(list(file_list)):
309 329
             if f.folder:
@@ -316,13 +336,11 @@ class Indexer():
316 336
             if dirname != last_dirname:
317 337
                 index = indexes.pop(dirname, None)
318 338
                 if index:
319
-                    newlist.append(index)
339
+                    new_list.append(index)
320 340
                     last_dirname = dirname
321
-            newlist.append(f)
322
-        newlist.reverse()
323
-        ret_file_list.file_list = newlist
324
-
325
-        return ret_file_list
341
+            new_list.append(f)
342
+        new_list.reverse()
343
+        file_list.file_list = new_list
326 344
 
327 345
     def make_index_file(self, folder_links, title):
328 346
         """Writes an index into a file for pushing"""
@@ -547,33 +565,33 @@ def run(cloud, container, files,
547 565
             prefix = '/'.join(parts[1:])
548 566
 
549 567
     # Create the objects to make sure the arguments are sound.
550
-    file_list = FileList()
551
-    indexer = Indexer(create_parent_links=parent_links,
552
-                      create_topdir_parent_link=topdir_parent_link,
553
-                      append_footer=footer)
554
-
555
-    # Scan the files.
556
-    for file_path in files:
557
-        file_list.add(file_path)
558
-
559
-    # (Possibly) make indexes.
560
-    if indexes:
561
-        file_list = indexer.make_indexes(file_list)
562
-
563
-    logging.debug("List of files prepared to upload:")
564
-    for x in file_list:
565
-        logging.debug(x)
566
-
567
-    # Do no connect to swift or do any uploading in a dry run
568
-    if dry_run:
569
-        # No URL is known, so return nothing
570
-        return
571
-
572
-    # Upload.
573
-    uploader = Uploader(cloud, container, prefix, delete_after,
574
-                        public)
575
-    uploader.upload(file_list)
576
-    return uploader.url
568
+    with FileList() as file_list:
569
+        indexer = Indexer(create_parent_links=parent_links,
570
+                          create_topdir_parent_link=topdir_parent_link,
571
+                          append_footer=footer)
572
+
573
+        # Scan the files.
574
+        for file_path in files:
575
+            file_list.add(file_path)
576
+
577
+        # (Possibly) make indexes.
578
+        if indexes:
579
+            indexer.make_indexes(file_list)
580
+
581
+        logging.debug("List of files prepared to upload:")
582
+        for x in file_list:
583
+            logging.debug(x)
584
+
585
+        # Do no connect to swift or do any uploading in a dry run
586
+        if dry_run:
587
+            # No URL is known, so return nothing
588
+            return
589
+
590
+        # Upload.
591
+        uploader = Uploader(cloud, container, prefix, delete_after,
592
+                            public)
593
+        uploader.upload(file_list)
594
+        return uploader.url
577 595
 
578 596
 
579 597
 def ansible_main():

Loading…
Cancel
Save