Browse Source

upload-logs-swift: Make indexer more generic

This converts the Indexer class from something that strictly generates
index.html files for folders to a more generic class that will be able
to hold other types of transformations we might like to do on the
total collections of uploaded files.

The index.html specific arguments are moved into make_indexes() and
the two helper functions that should not be called externally are
renamed private.

Change-Id: I388042ffb6a74c3200d92fb3a084369fcf2cf3a9
Ian Wienand 8 months ago
parent
commit
2d61e9d4fc

+ 8
- 8
roles/upload-logs-swift/library/test_zuul_swift_upload.py View File

@@ -257,10 +257,10 @@ class TestFileList(testtools.TestCase):
257 257
         '''Test index generation creates topdir parent link'''
258 258
         with FileList() as fl:
259 259
             fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
260
-            ix = Indexer(fl,
261
-                         create_parent_links=True,
262
-                         create_topdir_parent_link=True)
263
-            ix.make_indexes()
260
+            ix = Indexer(fl)
261
+            ix.make_indexes(
262
+                create_parent_links=True,
263
+                create_topdir_parent_link=True)
264 264
 
265 265
             self.assert_files(fl, [
266 266
                 ('', 'application/directory', None),
@@ -312,10 +312,10 @@ class TestFileList(testtools.TestCase):
312 312
         '''Test index generation creates topdir parent link'''
313 313
         with FileList() as fl:
314 314
             fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
315
-            ix = Indexer(fl,
316
-                         create_parent_links=False,
317
-                         create_topdir_parent_link=False)
318
-            ix.make_indexes()
315
+            ix = Indexer(fl)
316
+            ix.make_indexes(
317
+                create_parent_links=False,
318
+                create_topdir_parent_link=False)
319 319
 
320 320
             self.assert_files(fl, [
321 321
                 ('', 'application/directory', None),

+ 91
- 83
roles/upload-logs-swift/library/zuul_swift_upload.py View File

@@ -282,87 +282,23 @@ class FileList(Sequence):
282 282
 
283 283
 
284 284
 class Indexer():
285
-    """generates index.html files if requested."""
285
+    """Index a FileList
286 286
 
287
-    def __init__(self, file_list, create_parent_links=True,
288
-                 create_topdir_parent_link=False,
289
-                 append_footer='index_footer.html'):
287
+    Functions to generate indexes and other collated data for a
288
+    FileList
290 289
 
290
+     - make_indexes() : make index.html in folders
291
+    """
292
+    def __init__(self, file_list):
291 293
         '''
292 294
         Args:
293
-            file_list (FileList): A FileList object to be updated
294
-             with index files for each directory.
295
-            create_parent_links (bool):
296
-            create_topdir_parent_link (bool):
297
-            append_footer (str):
295
+            file_list (FileList): A FileList object with all files
296
+             to be indexed.
298 297
         '''
299 298
         assert isinstance(file_list, FileList)
300 299
         self.file_list = file_list
301
-        self.create_parent_links = create_parent_links
302
-        self.create_topdir_parent_link = create_topdir_parent_link
303
-        self.append_footer = append_footer
304
-        self.index_filename = 'index.html'
305
-
306
-    def make_indexes(self):
307
-        '''Make index files
308
-
309
-        Return:
310
-            No value, the self.file_list will be updated
311
-        '''
312
-        folders = collections.OrderedDict()
313
-        for f in self.file_list:
314
-            if f.folder:
315
-                folders[f.relative_path] = []
316
-                folder = os.path.dirname(os.path.dirname(
317
-                    f.relative_path + '/'))
318
-                if folder == '/':
319
-                    folder = ''
320
-            else:
321
-                folder = os.path.dirname(f.relative_path)
322
-            folders[folder].append(f)
323
-
324
-        indexes = {}
325
-        parent_file_detail = FileDetail(None, '..', '..')
326
-        for folder, files in folders.items():
327
-            # Don't add the pseudo-top-directory
328
-            if files and files[0].full_path is None:
329
-                files = files[1:]
330
-                if self.create_topdir_parent_link:
331
-                    files = [parent_file_detail] + files
332
-            elif self.create_parent_links:
333
-                files = [parent_file_detail] + files
334
-
335
-            # Do generate a link to the parent directory
336
-            full_path = self.make_index_file(files, 'Index of %s' % (folder,),
337
-                                             self.file_list.get_tempdir())
338 300
 
339
-            if full_path:
340
-                filename = os.path.basename(full_path)
341
-                relative_name = os.path.join(folder, filename)
342
-                indexes[folder] = FileDetail(full_path, relative_name)
343
-
344
-        # This appends the index file at the end of the group of files
345
-        # for each directory.
346
-        new_list = []
347
-        last_dirname = None
348
-        for f in reversed(list(self.file_list)):
349
-            if f.folder:
350
-                relative_path = f.relative_path + '/'
351
-            else:
352
-                relative_path = f.relative_path
353
-            dirname = os.path.dirname(relative_path)
354
-            if dirname == '/':
355
-                dirname = ''
356
-            if dirname != last_dirname:
357
-                index = indexes.pop(dirname, None)
358
-                if index:
359
-                    new_list.append(index)
360
-                    last_dirname = dirname
361
-            new_list.append(f)
362
-        new_list.reverse()
363
-        self.file_list.file_list = new_list
364
-
365
-    def make_index_file(self, folder_links, title, tempdir):
301
+    def _make_index_file(self, folder_links, title, tempdir, append_footer):
366 302
         """Writes an index into a file for pushing"""
367 303
         for file_details in folder_links:
368 304
             # Do not generate an index file if one exists already.
@@ -370,12 +306,13 @@ class Indexer():
370 306
             # content like python coverage info.
371 307
             if self.index_filename == file_details.filename:
372 308
                 return
373
-        index_content = self.generate_log_index(folder_links, title)
309
+        index_content = self._generate_log_index(
310
+            folder_links, title, append_footer)
374 311
         fd = open(os.path.join(tempdir, self.index_filename), 'w')
375 312
         fd.write(index_content)
376 313
         return os.path.join(tempdir, self.index_filename)
377 314
 
378
-    def generate_log_index(self, folder_links, title):
315
+    def _generate_log_index(self, folder_links, title, append_footer):
379 316
         """Create an index of logfiles and links to them"""
380 317
 
381 318
         output = '<html><head><title>%s</title></head><body>\n' % title
@@ -403,8 +340,8 @@ class Indexer():
403 340
             output += '<td style="text-align: right">%s</td>' % size
404 341
             output += '</tr>\n'
405 342
 
406
-            if (self.append_footer and
407
-                self.append_footer in file_details.filename):
343
+            if (append_footer and
344
+                append_footer in file_details.filename):
408 345
                 file_details_to_append = file_details
409 346
 
410 347
         output += '</table>'
@@ -420,6 +357,78 @@ class Indexer():
420 357
         output += '</body></html>\n'
421 358
         return output
422 359
 
360
+    def make_indexes(self, create_parent_links=True,
361
+                     create_topdir_parent_link=False,
362
+                     append_footer='index_footer.html'):
363
+        '''Make index.html files
364
+
365
+        Iterate the file list and crete index.html files for folders
366
+
367
+        Args:
368
+          create_parent_links (bool): Create parent links
369
+          create_topdir_parent_link (bool): Create topdir parent link
370
+          append_footer (str): Filename of a footer to append to each
371
+             generated page
372
+
373
+        Return:
374
+            No value, the self.file_list will be updated
375
+        '''
376
+        self.index_filename = 'index.html'
377
+
378
+        folders = collections.OrderedDict()
379
+        for f in self.file_list:
380
+            if f.folder:
381
+                folders[f.relative_path] = []
382
+                folder = os.path.dirname(os.path.dirname(
383
+                    f.relative_path + '/'))
384
+                if folder == '/':
385
+                    folder = ''
386
+            else:
387
+                folder = os.path.dirname(f.relative_path)
388
+            folders[folder].append(f)
389
+
390
+        indexes = {}
391
+        parent_file_detail = FileDetail(None, '..', '..')
392
+        for folder, files in folders.items():
393
+            # Don't add the pseudo-top-directory
394
+            if files and files[0].full_path is None:
395
+                files = files[1:]
396
+                if create_topdir_parent_link:
397
+                    files = [parent_file_detail] + files
398
+            elif create_parent_links:
399
+                files = [parent_file_detail] + files
400
+
401
+            # Do generate a link to the parent directory
402
+            full_path = self._make_index_file(files, 'Index of %s' % (folder,),
403
+                                              self.file_list.get_tempdir(),
404
+                                              append_footer)
405
+
406
+            if full_path:
407
+                filename = os.path.basename(full_path)
408
+                relative_name = os.path.join(folder, filename)
409
+                indexes[folder] = FileDetail(full_path, relative_name)
410
+
411
+        # This appends the index file at the end of the group of files
412
+        # for each directory.
413
+        new_list = []
414
+        last_dirname = None
415
+        for f in reversed(list(self.file_list)):
416
+            if f.folder:
417
+                relative_path = f.relative_path + '/'
418
+            else:
419
+                relative_path = f.relative_path
420
+            dirname = os.path.dirname(relative_path)
421
+            if dirname == '/':
422
+                dirname = ''
423
+            if dirname != last_dirname:
424
+                index = indexes.pop(dirname, None)
425
+                if index:
426
+                    new_list.append(index)
427
+                    last_dirname = dirname
428
+            new_list.append(f)
429
+        new_list.reverse()
430
+        self.file_list.file_list = new_list
431
+
423 432
 
424 433
 class DeflateFilter():
425 434
     chunk_size = 16384
@@ -585,18 +594,17 @@ def run(cloud, container, files,
585 594
 
586 595
     # Create the objects to make sure the arguments are sound.
587 596
     with FileList() as file_list:
588
-        indexer = Indexer(file_list,
589
-                          create_parent_links=parent_links,
590
-                          create_topdir_parent_link=topdir_parent_link,
591
-                          append_footer=footer)
592
-
593 597
         # Scan the files.
594 598
         for file_path in files:
595 599
             file_list.add(file_path)
596 600
 
601
+        indexer = Indexer(file_list)
602
+
597 603
         # (Possibly) make indexes.
598 604
         if indexes:
599
-            indexer.make_indexes()
605
+            indexer.make_indexes(create_parent_links=parent_links,
606
+                                 create_topdir_parent_link=topdir_parent_link,
607
+                                 append_footer=footer)
600 608
 
601 609
         logging.debug("List of files prepared to upload:")
602 610
         for x in file_list:

Loading…
Cancel
Save