From 76d0a375624227fa036c76b686d25532f422de61 Mon Sep 17 00:00:00 2001 From: Lance Xu Date: Tue, 5 Dec 2023 11:38:42 -0500 Subject: [PATCH] Add rendering collect bundle files to report tool This update adds rendering support of the collect bundle itself. The user can navigate the collect bundle files from a browser. The collect bundle menu shows after the system info and result section. Opening the menu hosts collects that are part of the collect bundle. Clicking each bundle opens a new tab showing the corresponding bundle. The layout for collect bundle will be the same as the results section. Test Plan: PASS: Verify the menu and content panel can be adjusted horizontally PASS: Verify the collect bundle section is shown under results section PASS: Verify collect bundle menu can show/hide PASS: Verify all collect bundle items are shown PASS: Verify clicking an item leads to a new tab showing the bundle PASS: Verify menus are levelled with '+'/'-' function to show/hide PASS: Verify menus are colored dark green and items are light green PASS: Verify empty folders are showing grey with disabled click PASS: Verify empty files are showing grey with disabled click PASS: Verify files that does not have permission are showing grey PASS: Verify '.log', '.conf', '.info', '.json', '.alarm', '.pid', '.list', '.lock', '.txt' files can be directly viewed when opened PASS: Verify handling of files that are not in the above extension PASS: Verify a new tab is opened if the file is viewable PASS: Verify a download popup is opened if the file is not viewable PASS: Verify index.html is in a reasonable size PASS: Verify index.html loading does not get stuck PASS: Verify the generated html with css content has no error in console Story: 2010533 Task: 49191 Change-Id: I71c4c6b39ca68464baf09c7d1708348e30989fda Signed-off-by: Lance Xu --- tools/collector/debian-scripts/report/README | 9 + .../collector/debian-scripts/report/render.py | 250 ++++++++++++++++-- 2 files changed, 239 insertions(+), 20 deletions(-) diff --git a/tools/collector/debian-scripts/report/README b/tools/collector/debian-scripts/report/README index fda3e274..ac14d30b 100644 --- a/tools/collector/debian-scripts/report/README +++ b/tools/collector/debian-scripts/report/README @@ -210,3 +210,12 @@ Failures : 4 /localdisk/CGTS-44887/ALL_NODES_20230307.183540/report_analysi Inspect the Correlated and Plugin results files for failures, alarms, events and state changes. + +The report analysis and collect bundle can be viewed in a html browser by loading the +index.html file created when the report tool is run. + +The rendering tool is displayed with a menu-content layout. +There will be three sections: System Info, Result, Collect Bundle. +System Info contains controller, storage, and worker. +controller-0 is shown by default. +Users can click '+'/'-' in menu to show/hide system info contents in content panel. diff --git a/tools/collector/debian-scripts/report/render.py b/tools/collector/debian-scripts/report/render.py index 72aae54b..881b3580 100755 --- a/tools/collector/debian-scripts/report/render.py +++ b/tools/collector/debian-scripts/report/render.py @@ -6,17 +6,33 @@ # ######################################################################## # -# This file contains the Render function -# The Rendering tool visualizes the collect bundle and generates index.html file +# This file contains the Render function. +# The Rendering tool visualizes the collect bundle and generates +# an index.html file # ######################################################################## from datetime import datetime import os +from pathlib import Path +import re + + +def can_open_file(file_path): + """Test if the file can be opened or not empty + + Parameters: + file_path(Path): path of the file + """ + try: + with open(file_path, 'r'): + return os.path.getsize(file_path) != 0 + except IOError: + return False def extract_section(log_contents, start_phrase): - """extract the correlated or plugin content of the summary + """Extract the correlated or plugin content of the summary Parameters: log_contents (string): content of the log @@ -32,7 +48,7 @@ def extract_section(log_contents, start_phrase): def remove_timestamp(text): - """remove timestamp of summary message + """Remove timestamp of summary message Parameters: text (string): the summary message @@ -51,7 +67,7 @@ def remove_timestamp(text): def remove_emptyinfo(text): - """ remove 'INFO' text of summary message + """Remove 'INFO' text of summary message Parameters: text (string): the summary message @@ -66,7 +82,7 @@ def remove_emptyinfo(text): def process_section(section, title): - """return text with timestamp and INFO: removed + """Return text with timestamp and INFO: removed Parameters: section (string): the message of the correlated/plugins section @@ -79,7 +95,7 @@ def process_section(section, title): def classify_node(data): - """classify node type in system_info summary + """Classify node type in system_info summary Parameters: data (string): the summary of system_info @@ -92,7 +108,7 @@ def classify_node(data): def controller_sort(x): - """sort the controller, place the controller-0 first + """Sort the controller, place the controller-0 first Parameters: x (list): list of controller info @@ -101,7 +117,7 @@ def controller_sort(x): def html_css(): - """static css code of the rendering tool + """Static css code of the rendering tool iframe, textarea: the content panel showing information #show-worker: the show more worker button @@ -132,7 +148,7 @@ def html_css(): .container-menu {{ display: grid; - grid-template-columns: 25% 75%; + grid-template-columns: 25% auto 1fr; grid-gap: 10px; background-color: #f0f0f0; }} @@ -166,6 +182,7 @@ def html_css(): }} .menuTitle {{ + font-weight: bold; color: #00857e !important; }} @@ -174,6 +191,12 @@ def html_css(): display: flex; }} + .resizer {{ + width: 10px; + background: #ccc; + cursor: ew-resize; + }} + .menuItem .icon {{ margin-right: 5px; }} @@ -193,7 +216,7 @@ def html_css(): margin-top: 10px; }} - .content-item, .content-itemtwo {{ + .content-item, .content-itemtwo, .content-itemthree {{ display: none; }} @@ -218,6 +241,31 @@ def html_css(): color: #2F4F4F; }} + .caret {{ + cursor: pointer; + user-select: none; + color: #00857e; + font-weight: bold; + }} + + .caret::before {{ + content: '+'; + color: #2F4F4F; + margin-right: 6px; + }} + + .caret-down::before {{ + color: #2F4F4F; + content: '-'; + }} + + .text-color {{ + color: #00ada4; + }} + + .nested {{ display: none; }} + .active {{ display: block; }} + """ @@ -225,7 +273,7 @@ def html_css(): def html_script(): - """static script code + """Static script code Functions: toggleContent: show content in System Info section @@ -234,6 +282,7 @@ def html_script(): showContentStorage: display content of selected storage item showContentWorker: display content of selected worker item showContentTwo: display content of result section + toggleTree: show the collect bundle """ html_content_script = """ """ @@ -402,7 +537,7 @@ def html_script(): def html_info(sys_section): - """system info part generation + """System info part generation reads from plugin/system_info and show by different types order: controller, storage(if there exists), worker(if there exists) @@ -440,7 +575,7 @@ def html_info(sys_section): html_content_one += """ -
+
" - html_content_one += """
""" + html_content_one += """
""" # controller-0 html_content_one += """
""" @@ -505,7 +640,7 @@ def html_info(sys_section): def html_result(log_contents, output_dir): - """result part generation in the menu-content style + """Result part generation in the menu-content style generates correlated results, plugin results, and the items under them subitems for plugins and correlated results under separate menus @@ -540,7 +675,7 @@ def html_result(log_contents, output_dir): html_content_two = "" html_content_two += """ -
+
  • @@ -562,7 +697,7 @@ def html_result(log_contents, output_dir): for item in plugin_items: html_content_two += f'
  • {item["name"]}
    • ' - html_content_two += """
""" + html_content_two += "
" + generate_collect() + "
" html_content_two += """
""" for item in correlated_items: @@ -576,12 +711,86 @@ def html_result(log_contents, output_dir): html_content_two += """
- """ return html_content_two +def generate_collect(): + os.chdir('../../') + current_directory = Path('.') + finalstr = """
  • + + + Collect Bundle
    • " + return finalstr + + +def html_collect(): + """Collect bundle code generation + + Calls a helper function to to generate the collect bundle + """ + current_directory = Path('.') + tree_html = "" + content_html = "
    " + target_dir = current_directory.resolve() + newtree_html, newcontent_html = generate_directory_tree(current_directory, target_dir, 0) + tree_html += newtree_html + content_html += newcontent_html + content_html += "
    " + html_content_three = """
    + """ + tree_html + "
    " + content_html + "
    " + return html_content_three + + +def generate_directory_tree(directory_path, target_dir, is_top_level): + """Helper function for Collect bundle generation + + Parameters: + directory_path(Path): the path of the directory in each call + target_dir(string): the path of the file/folder + is_top_level(bool): if the level is the top level of the collect bundle + """ + directory_name = directory_path.name + tree_html = "" + content_html = "" + approved_list = ['.log', '.conf', '.info', '.json', '.alarm', '.pid', '.list', '.lock', '.txt'] + if is_top_level == 1: + temp_name = re.sub(r'[^a-zA-Z0-9]', '', directory_name) + tree_html = f'
  • {directory_name}
      ' + if is_top_level > 1: + tree_html = f'
    • {directory_name}
        ' + if is_top_level < 5: + for item in directory_path.iterdir(): + try: + if item.is_dir() and item.name != "report_analysis": + nested_tree_html, nested_content_html = generate_directory_tree(item, target_dir, is_top_level + 1) + tree_html += nested_tree_html + content_html += nested_content_html + elif item.is_file(): + if not can_open_file(item): + tree_html += f'
      • {item}
        • ' + else: + if item.name.endswith(tuple(approved_list)): + tree_html += f'
      • {item.name}
        • ' + content_html += f'

        {item.name}

        ' + else: + if not item.name.endswith(".tgz"): + tree_html += f'
      • {item}
        • ' + # if it's permission error, just skip reading the file or folder + except PermissionError as e: + continue + if is_top_level: + tree_html += '
      • ' + + return tree_html, content_html + + # main def main(input_dir, output_dir): reportlog_path = os.path.join(output_dir, 'report.log') @@ -596,8 +805,9 @@ def main(input_dir, output_dir): html_file = os.path.abspath(os.path.join(output_dir, 'index.html')) sys_section = sysinfo_contents.strip().split("\n\n") - html_content = html_css() + html_info(sys_section) + html_result(log_contents, output_dir) + html_script() + html_content = html_css() + html_info(sys_section) + html_result(log_contents, output_dir) html_content = html_content.format() + html_content += html_collect() + html_script() # write the HTML content to file with open(html_file, "w") as file: