From 8937e1bb0e041f4ddc07475260e514daa608ea7f Mon Sep 17 00:00:00 2001 From: Lance Xu Date: Tue, 5 Dec 2023 11:38:42 -0500 Subject: [PATCH] Add rendering collect bundle files to report tool This update adds rendering support of the collect bundle itself. The user can navigate the collect bundle files from a browser. The collect bundle menu shows after the system info and result section. Opening the menu hosts collects that are part of the collect bundle. Clicking each bundle opens a new tab showing the corresponding bundle. The layout for collect bundle will be the same as the results section. Test Plan: PASS: Verify the menu and content panel can be adjusted horizontally PASS: Verify the collect bundle section is shown under results section PASS: Verify collect bundle menu can show/hide PASS: Verify all collect bundle items are shown PASS: Verify clicking an item leads to a new tab showing the bundle PASS: Verify menus are levelled with '+'/'-' function to show/hide PASS: Verify menus are colored dark green and items are light green PASS: Verify empty folders are showing grey with disabled click PASS: Verify empty files are showing grey with disabled click PASS: Verify files that does not have permission are showing grey PASS: Verify '.log', '.conf', '.info', '.json', '.alarm', '.pid', '.list', '.lock', '.txt' files can be directly viewed when opened PASS: Verify handling of files that are not in the above extension PASS: Verify a new tab is opened if the file is viewable PASS: Verify a download popup is opened if the file is not viewable PASS: Verify index.html is in a reasonable size PASS: Verify index.html loading does not get stuck PASS: Verify the generated html with css content has no error in console Story: 2010533 Task: 49191 Change-Id: I71c4c6b39ca68464baf09c7d1708348e30989fda Signed-off-by: Lance Xu --- tools/collector/debian-scripts/report/README | 36 +++ .../debian-scripts/report/render.exclude | 41 +++ .../collector/debian-scripts/report/render.py | 273 ++++++++++++++++-- tools/collector/debian/deb_folder/rules | 1 + 4 files changed, 329 insertions(+), 22 deletions(-) create mode 100755 tools/collector/debian-scripts/report/render.exclude diff --git a/tools/collector/debian-scripts/report/README b/tools/collector/debian-scripts/report/README index fda3e274..93890396 100644 --- a/tools/collector/debian-scripts/report/README +++ b/tools/collector/debian-scripts/report/README @@ -210,3 +210,39 @@ Failures : 4 /localdisk/CGTS-44887/ALL_NODES_20230307.183540/report_analysi Inspect the Correlated and Plugin results files for failures, alarms, events and state changes. + +The report analysis and collect bundle can be viewed in a html browser by loading the +index.html file created when the report tool is run. + +The rendering tool is displayed with a menu-content layout. +There will be three sections: System Info, Result, Collect Bundle. +System Info contains controller, storage, and worker. +controller-0 is shown by default. +Users can click '+'/'-' in menu to show/hide system info contents in content panel. + +System Info controller-0 + ------------ +- controller-0 System Type: All-in-one ++ controller-1 S/W Version: 22.12 +------------ System Node: duplex +- Storage DC Role : systemcontroller ++ storage-0 Node Type : Controller ++ storage-1 subfunction: controller +------------ Mgmt Iface : vlan166 +- Workers Clstr Iface: vlan167 ++ compute-0 Build Type : formal ++ compute-1 Build Date : 2022-12-19 07:22:00 +0000 + controllers: controller-1, controller-0 + workers : compute-0, compute-1 + +Result section contains Correlated Results and Plugin Results. +Both Correlated Results and Plugin Results have their subitems. +Collect Bundle is shown after the Result section. +Opening the Collect Bundle menu shows all the collect bundle items. +Clicking a bundle will open the new tab corresponding to that bundle. +If the folder or file is empty or does not have permission to open, it will show grey color. +Files that are not empty will show light green color. +File content is shown on the right panel, as previous layouts. +For files that does not have a specific extension, a new tab will be opened. +If they cannot be opened, a download popup will be shown, or it will be directly downloaded, +depending on the browser settings. diff --git a/tools/collector/debian-scripts/report/render.exclude b/tools/collector/debian-scripts/report/render.exclude new file mode 100755 index 00000000..9990b4a2 --- /dev/null +++ b/tools/collector/debian-scripts/report/render.exclude @@ -0,0 +1,41 @@ +/etc/apache2 +/etc/apparmor.d +/etc/apt +/etc/cloud +/etc/collectd.d +/etc/default +/etc/dhcp +/etc/fonts +/etc/init.d +/etc/ldap +/etc/ldapscripts +/etc/libvirt +/etc/lighttpd +/etc/logcheck +/etc/lvm +/etc/modules-load.d +/etc/openstack-dashboard +/etc/pmon.d +/etc/pycadf +/etc/qat +/etc/rc0.d +/etc/rc3.d +/etc/rc6.d +/etc/rcS.d +/etc/runit +/etc/strongswan.d +/etc/sv +/etc/sysctl.d +/etc/systems +/etc/tmpfiles.d +/etc/tuned +/etc/update-motd.d +/etc/upgrade.d +/etc/vswitch +/root/.rpmdb +/var/run/calico +/var/run/dpdk/vs +/var/run/libvirt +/var/run/netns +/var/run/systems +/var/run/udev diff --git a/tools/collector/debian-scripts/report/render.py b/tools/collector/debian-scripts/report/render.py index 72aae54b..09574337 100755 --- a/tools/collector/debian-scripts/report/render.py +++ b/tools/collector/debian-scripts/report/render.py @@ -6,17 +6,45 @@ # ######################################################################## # -# This file contains the Render function -# The Rendering tool visualizes the collect bundle and generates index.html file +# This file contains the Render function. +# The Rendering tool visualizes the collect bundle and generates +# an index.html file # ######################################################################## from datetime import datetime import os +from pathlib import Path +import re + + +def exclude_path(): + """Generate a set for files to be excluded + + """ + exclude_file = Path(__file__).parent / 'render.exclude' + exclude_paths = set() + if exclude_file.exists(): + with exclude_file.open('r') as file: + exclude_paths = set(line.strip() for line in file) + return exclude_paths + + +def can_open_file(file_path): + """Test if the file can be opened or not empty + + Parameters: + file_path(Path): path of the file + """ + try: + with open(file_path, 'r'): + return os.path.getsize(file_path) != 0 + except IOError: + return False def extract_section(log_contents, start_phrase): - """extract the correlated or plugin content of the summary + """Extract the correlated or plugin content of the summary Parameters: log_contents (string): content of the log @@ -32,7 +60,7 @@ def extract_section(log_contents, start_phrase): def remove_timestamp(text): - """remove timestamp of summary message + """Remove timestamp of summary message Parameters: text (string): the summary message @@ -51,7 +79,7 @@ def remove_timestamp(text): def remove_emptyinfo(text): - """ remove 'INFO' text of summary message + """Remove 'INFO' text of summary message Parameters: text (string): the summary message @@ -66,7 +94,7 @@ def remove_emptyinfo(text): def process_section(section, title): - """return text with timestamp and INFO: removed + """Return text with timestamp and INFO: removed Parameters: section (string): the message of the correlated/plugins section @@ -79,7 +107,7 @@ def process_section(section, title): def classify_node(data): - """classify node type in system_info summary + """Classify node type in system_info summary Parameters: data (string): the summary of system_info @@ -92,7 +120,7 @@ def classify_node(data): def controller_sort(x): - """sort the controller, place the controller-0 first + """Sort the controller, place the controller-0 first Parameters: x (list): list of controller info @@ -101,7 +129,7 @@ def controller_sort(x): def html_css(): - """static css code of the rendering tool + """Static css code of the rendering tool iframe, textarea: the content panel showing information #show-worker: the show more worker button @@ -132,7 +160,7 @@ def html_css(): .container-menu {{ display: grid; - grid-template-columns: 25% 75%; + grid-template-columns: 25% auto 1fr; grid-gap: 10px; background-color: #f0f0f0; }} @@ -161,11 +189,14 @@ def html_css(): text-decoration: none; color: #00ada4; font-weight: bold; + font-family: Courier; display: block; padding: 6px 10px; }} .menuTitle {{ + font-weight: bold; + font-family: Courier; color: #00857e !important; }} @@ -174,6 +205,12 @@ def html_css(): display: flex; }} + .resizer {{ + width: 10px; + background: #ccc; + cursor: ew-resize; + }} + .menuItem .icon {{ margin-right: 5px; }} @@ -193,7 +230,7 @@ def html_css(): margin-top: 10px; }} - .content-item, .content-itemtwo {{ + .content-item, .content-itemtwo, .content-itemthree {{ display: none; }} @@ -218,6 +255,31 @@ def html_css(): color: #2F4F4F; }} + .caret {{ + cursor: pointer; + user-select: none; + color: #00857e; + font-weight: bold; + }} + + .caret::before {{ + content: '+'; + color: #2F4F4F; + margin-right: 6px; + }} + + .caret-down::before {{ + color: #2F4F4F; + content: '-'; + }} + + .text-color {{ + color: #00ada4; + }} + + .nested {{ display: none; }} + .active {{ display: block; }} + """ @@ -225,7 +287,7 @@ def html_css(): def html_script(): - """static script code + """Static script code Functions: toggleContent: show content in System Info section @@ -234,6 +296,7 @@ def html_script(): showContentStorage: display content of selected storage item showContentWorker: display content of selected worker item showContentTwo: display content of result section + toggleTree: show the collect bundle """ html_content_script = """ """ @@ -402,7 +551,7 @@ def html_script(): def html_info(sys_section): - """system info part generation + """System info part generation reads from plugin/system_info and show by different types order: controller, storage(if there exists), worker(if there exists) @@ -440,7 +589,7 @@ def html_info(sys_section): html_content_one += """ -
+
    System Information @@ -480,12 +629,12 @@ def html_info(sys_section): html_content_one += "
" - html_content_one += """
""" + html_content_one += """
""" # controller-0 html_content_one += """
""" for i in controller_zero: - html_content_one += f'{i}' + html_content_one += f'{i}'.replace(' ', ' ') html_content_one += "
" html_content_one += "
" @@ -497,7 +646,8 @@ def html_info(sys_section): div_id = f"{section_type}-{i}" html_content_one += f'" html_content_one += "

""" @@ -505,7 +655,7 @@ def html_info(sys_section): def html_result(log_contents, output_dir): - """result part generation in the menu-content style + """Result part generation in the menu-content style generates correlated results, plugin results, and the items under them subitems for plugins and correlated results under separate menus @@ -540,7 +690,7 @@ def html_result(log_contents, output_dir): html_content_two = "" html_content_two += """ -
+
  • @@ -562,7 +712,7 @@ def html_result(log_contents, output_dir): for item in plugin_items: html_content_two += f'
  • {item["name"]}
    • ' - html_content_two += """
""" + html_content_two += "
" + generate_collect() + "
" html_content_two += """
""" for item in correlated_items: @@ -576,12 +726,90 @@ def html_result(log_contents, output_dir): html_content_two += """
- """ return html_content_two +def generate_collect(): + os.chdir('../../') + current_directory = Path('.') + finalstr = """
  • + + + Collect Bundle
    • " + return finalstr + + +def html_collect(): + """Collect bundle code generation + + Calls a helper function to to generate the collect bundle + """ + current_directory = Path('.') + tree_html = "" + content_html = "
    " + target_dir = current_directory.resolve() + newtree_html, newcontent_html = generate_directory_tree(current_directory, exclude_path(), target_dir, 0) + tree_html += newtree_html + content_html += newcontent_html + content_html += "
    " + html_content_three = """
    + """ + tree_html + "
    " + content_html + "
    " + return html_content_three + + +def generate_directory_tree(directory_path, exclude_path, target_dir, is_top_level): + """Helper function for Collect bundle generation + + Parameters: + directory_path(Path): the path of the directory in each call + target_dir(string): the path of the file/folder + is_top_level(bool): if the level is the top level of the collect bundle + """ + directory_name = directory_path.name + tree_html = "" + content_html = "" + approved_list = ['.log', '.conf', '.info', '.json', '.alarm', '.pid', '.list', '.lock', '.txt'] + if is_top_level == 1: + temp_name = re.sub(r'[^a-zA-Z0-9]', '', directory_name) + tree_html = f'
  • {directory_name}
      ' + if is_top_level > 1: + tree_html = f'
    • {directory_name}
        ' + for item in directory_path.iterdir(): + # write to a file called 'exclude', all the files including the full path + # if item in exclude, do not add to html + # else add it to another + item_path = str(item) + if not any(exclude_item in item_path for exclude_item in exclude_path): + try: + if item.is_dir() and item.name != "report_analysis": + nested_tree_html, nested_content_html = generate_directory_tree(item, exclude_path, target_dir, is_top_level + 1) + tree_html += nested_tree_html + content_html += nested_content_html + elif item.is_file(): + if not can_open_file(item): + tree_html += f'
      • {item}
        • ' + else: + if item.name.endswith(tuple(approved_list)): + tree_html += f'
      • {item.name}
        • ' + content_html += f'

        {item.name}

        ' + else: + if not item.name.endswith(".tgz") and not item.name.endswith(".gz"): + tree_html += f'
      • {item}
        • ' + # if it's permission error, just skip reading the file or folder + except PermissionError as e: + continue + if is_top_level: + tree_html += '
      • ' + + return tree_html, content_html + + # main def main(input_dir, output_dir): reportlog_path = os.path.join(output_dir, 'report.log') @@ -596,8 +824,9 @@ def main(input_dir, output_dir): html_file = os.path.abspath(os.path.join(output_dir, 'index.html')) sys_section = sysinfo_contents.strip().split("\n\n") - html_content = html_css() + html_info(sys_section) + html_result(log_contents, output_dir) + html_script() + html_content = html_css() + html_info(sys_section) + html_result(log_contents, output_dir) html_content = html_content.format() + html_content += html_collect() + html_script() # write the HTML content to file with open(html_file, "w") as file: diff --git a/tools/collector/debian/deb_folder/rules b/tools/collector/debian/deb_folder/rules index 318da0c9..4c56fe23 100755 --- a/tools/collector/debian/deb_folder/rules +++ b/tools/collector/debian/deb_folder/rules @@ -39,6 +39,7 @@ override_dh_auto_install: install -m 755 -p report/plugin.py $(ROOT)/usr/local/bin/report/plugin.py install -m 755 -p report/correlator.py $(ROOT)/usr/local/bin/report/correlator.py install -m 755 -p report/render.py $(ROOT)/usr/local/bin/report/render.py + install -m 755 -p report/render.exclude $(ROOT)/usr/local/bin/report/render.exclude install -m 644 -p report/README $(ROOT)/usr/local/bin/report/README # Report Tool Plugin Algorithms