Browse Source

Merge "upload-logs-swift: Make indexer more generic"

Zuul 4 months ago
parent
commit
2ea7a1b6bd

+ 8
- 8
roles/upload-logs-swift/library/test_zuul_swift_upload.py View File

@@ -257,10 +257,10 @@ class TestFileList(testtools.TestCase):
257 257
         '''Test index generation creates topdir parent link'''
258 258
         with FileList() as fl:
259 259
             fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
260
-            ix = Indexer(fl,
261
-                         create_parent_links=True,
262
-                         create_topdir_parent_link=True)
263
-            ix.make_indexes()
260
+            ix = Indexer(fl)
261
+            ix.make_indexes(
262
+                create_parent_links=True,
263
+                create_topdir_parent_link=True)
264 264
 
265 265
             self.assert_files(fl, [
266 266
                 ('', 'application/directory', None),
@@ -312,10 +312,10 @@ class TestFileList(testtools.TestCase):
312 312
         '''Test index generation creates topdir parent link'''
313 313
         with FileList() as fl:
314 314
             fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
315
-            ix = Indexer(fl,
316
-                         create_parent_links=False,
317
-                         create_topdir_parent_link=False)
318
-            ix.make_indexes()
315
+            ix = Indexer(fl)
316
+            ix.make_indexes(
317
+                create_parent_links=False,
318
+                create_topdir_parent_link=False)
319 319
 
320 320
             self.assert_files(fl, [
321 321
                 ('', 'application/directory', None),

+ 91
- 83
roles/upload-logs-swift/library/zuul_swift_upload.py View File

@@ -282,87 +282,23 @@ class FileList(Sequence):
282 282
 
283 283
 
284 284
 class Indexer():
285
-    """generates index.html files if requested."""
285
+    """Index a FileList
286 286
 
287
-    def __init__(self, file_list, create_parent_links=True,
288
-                 create_topdir_parent_link=False,
289
-                 append_footer='index_footer.html'):
287
+    Functions to generate indexes and other collated data for a
288
+    FileList
290 289
 
290
+     - make_indexes() : make index.html in folders
291
+    """
292
+    def __init__(self, file_list):
291 293
         '''
292 294
         Args:
293
-            file_list (FileList): A FileList object to be updated
294
-             with index files for each directory.
295
-            create_parent_links (bool):
296
-            create_topdir_parent_link (bool):
297
-            append_footer (str):
295
+            file_list (FileList): A FileList object with all files
296
+             to be indexed.
298 297
         '''
299 298
         assert isinstance(file_list, FileList)
300 299
         self.file_list = file_list
301
-        self.create_parent_links = create_parent_links
302
-        self.create_topdir_parent_link = create_topdir_parent_link
303
-        self.append_footer = append_footer
304
-        self.index_filename = 'index.html'
305
-
306
-    def make_indexes(self):
307
-        '''Make index files
308
-
309
-        Return:
310
-            No value, the self.file_list will be updated
311
-        '''
312
-        folders = collections.OrderedDict()
313
-        for f in self.file_list:
314
-            if f.folder:
315
-                folders[f.relative_path] = []
316
-                folder = os.path.dirname(os.path.dirname(
317
-                    f.relative_path + '/'))
318
-                if folder == '/':
319
-                    folder = ''
320
-            else:
321
-                folder = os.path.dirname(f.relative_path)
322
-            folders[folder].append(f)
323
-
324
-        indexes = {}
325
-        parent_file_detail = FileDetail(None, '..', '..')
326
-        for folder, files in folders.items():
327
-            # Don't add the pseudo-top-directory
328
-            if files and files[0].full_path is None:
329
-                files = files[1:]
330
-                if self.create_topdir_parent_link:
331
-                    files = [parent_file_detail] + files
332
-            elif self.create_parent_links:
333
-                files = [parent_file_detail] + files
334
-
335
-            # Do generate a link to the parent directory
336
-            full_path = self.make_index_file(files, 'Index of %s' % (folder,),
337
-                                             self.file_list.get_tempdir())
338 300
 
339
-            if full_path:
340
-                filename = os.path.basename(full_path)
341
-                relative_name = os.path.join(folder, filename)
342
-                indexes[folder] = FileDetail(full_path, relative_name)
343
-
344
-        # This appends the index file at the end of the group of files
345
-        # for each directory.
346
-        new_list = []
347
-        last_dirname = None
348
-        for f in reversed(list(self.file_list)):
349
-            if f.folder:
350
-                relative_path = f.relative_path + '/'
351
-            else:
352
-                relative_path = f.relative_path
353
-            dirname = os.path.dirname(relative_path)
354
-            if dirname == '/':
355
-                dirname = ''
356
-            if dirname != last_dirname:
357
-                index = indexes.pop(dirname, None)
358
-                if index:
359
-                    new_list.append(index)
360
-                    last_dirname = dirname
361
-            new_list.append(f)
362
-        new_list.reverse()
363
-        self.file_list.file_list = new_list
364
-
365
-    def make_index_file(self, folder_links, title, tempdir):
301
+    def _make_index_file(self, folder_links, title, tempdir, append_footer):
366 302
         """Writes an index into a file for pushing"""
367 303
         for file_details in folder_links:
368 304
             # Do not generate an index file if one exists already.
@@ -370,12 +306,13 @@ class Indexer():
370 306
             # content like python coverage info.
371 307
             if self.index_filename == file_details.filename:
372 308
                 return
373
-        index_content = self.generate_log_index(folder_links, title)
309
+        index_content = self._generate_log_index(
310
+            folder_links, title, append_footer)
374 311
         fd = open(os.path.join(tempdir, self.index_filename), 'w')
375 312
         fd.write(index_content)
376 313
         return os.path.join(tempdir, self.index_filename)
377 314
 
378
-    def generate_log_index(self, folder_links, title):
315
+    def _generate_log_index(self, folder_links, title, append_footer):
379 316
         """Create an index of logfiles and links to them"""
380 317
 
381 318
         output = '<html><head><title>%s</title></head><body>\n' % title
@@ -403,8 +340,8 @@ class Indexer():
403 340
             output += '<td style="text-align: right">%s</td>' % size
404 341
             output += '</tr>\n'
405 342
 
406
-            if (self.append_footer and
407
-                self.append_footer in file_details.filename):
343
+            if (append_footer and
344
+                append_footer in file_details.filename):
408 345
                 file_details_to_append = file_details
409 346
 
410 347
         output += '</table>'
@@ -420,6 +357,78 @@ class Indexer():
420 357
         output += '</body></html>\n'
421 358
         return output
422 359
 
360
+    def make_indexes(self, create_parent_links=True,
361
+                     create_topdir_parent_link=False,
362
+                     append_footer='index_footer.html'):
363
+        '''Make index.html files
364
+
365
+        Iterate the file list and crete index.html files for folders
366
+
367
+        Args:
368
+          create_parent_links (bool): Create parent links
369
+          create_topdir_parent_link (bool): Create topdir parent link
370
+          append_footer (str): Filename of a footer to append to each
371
+             generated page
372
+
373
+        Return:
374
+            No value, the self.file_list will be updated
375
+        '''
376
+        self.index_filename = 'index.html'
377
+
378
+        folders = collections.OrderedDict()
379
+        for f in self.file_list:
380
+            if f.folder:
381
+                folders[f.relative_path] = []
382
+                folder = os.path.dirname(os.path.dirname(
383
+                    f.relative_path + '/'))
384
+                if folder == '/':
385
+                    folder = ''
386
+            else:
387
+                folder = os.path.dirname(f.relative_path)
388
+            folders[folder].append(f)
389
+
390
+        indexes = {}
391
+        parent_file_detail = FileDetail(None, '..', '..')
392
+        for folder, files in folders.items():
393
+            # Don't add the pseudo-top-directory
394
+            if files and files[0].full_path is None:
395
+                files = files[1:]
396
+                if create_topdir_parent_link:
397
+                    files = [parent_file_detail] + files
398
+            elif create_parent_links:
399
+                files = [parent_file_detail] + files
400
+
401
+            # Do generate a link to the parent directory
402
+            full_path = self._make_index_file(files, 'Index of %s' % (folder,),
403
+                                              self.file_list.get_tempdir(),
404
+                                              append_footer)
405
+
406
+            if full_path:
407
+                filename = os.path.basename(full_path)
408
+                relative_name = os.path.join(folder, filename)
409
+                indexes[folder] = FileDetail(full_path, relative_name)
410
+
411
+        # This appends the index file at the end of the group of files
412
+        # for each directory.
413
+        new_list = []
414
+        last_dirname = None
415
+        for f in reversed(list(self.file_list)):
416
+            if f.folder:
417
+                relative_path = f.relative_path + '/'
418
+            else:
419
+                relative_path = f.relative_path
420
+            dirname = os.path.dirname(relative_path)
421
+            if dirname == '/':
422
+                dirname = ''
423
+            if dirname != last_dirname:
424
+                index = indexes.pop(dirname, None)
425
+                if index:
426
+                    new_list.append(index)
427
+                    last_dirname = dirname
428
+            new_list.append(f)
429
+        new_list.reverse()
430
+        self.file_list.file_list = new_list
431
+
423 432
 
424 433
 class DeflateFilter():
425 434
     chunk_size = 16384
@@ -585,18 +594,17 @@ def run(cloud, container, files,
585 594
 
586 595
     # Create the objects to make sure the arguments are sound.
587 596
     with FileList() as file_list:
588
-        indexer = Indexer(file_list,
589
-                          create_parent_links=parent_links,
590
-                          create_topdir_parent_link=topdir_parent_link,
591
-                          append_footer=footer)
592
-
593 597
         # Scan the files.
594 598
         for file_path in files:
595 599
             file_list.add(file_path)
596 600
 
601
+        indexer = Indexer(file_list)
602
+
597 603
         # (Possibly) make indexes.
598 604
         if indexes:
599
-            indexer.make_indexes()
605
+            indexer.make_indexes(create_parent_links=parent_links,
606
+                                 create_topdir_parent_link=topdir_parent_link,
607
+                                 append_footer=footer)
600 608
 
601 609
         logging.debug("List of files prepared to upload:")
602 610
         for x in file_list:

Loading…
Cancel
Save