From 70ed60165628f0b8f3eaa7ecf4de559ed9976c8b Mon Sep 17 00:00:00 2001 From: Vitor Vidal Date: Fri, 28 Nov 2025 16:47:29 -0300 Subject: [PATCH] Adding metric reporter keyword for running metric reporter tool. Metric reporter tool allows to generate kpi data used by system test scenarios and report creation. Updating test_container_deployment.py to execute metric reporter script at the end of each test and save the contents to the local host. Change-Id: I9fa4964f27ad8100552c09cb18a44c06f295b911 Signed-off-by: Vitor Vidal --- config/system_test/metric_report_server.json5 | 10 + .../system_test/metric_reporter_keywords.py | 173 ++++++++++++++++++ .../system_test/test_container_deployment.py | 18 +- 3 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 config/system_test/metric_report_server.json5 create mode 100644 keywords/system_test/metric_reporter_keywords.py diff --git a/config/system_test/metric_report_server.json5 b/config/system_test/metric_report_server.json5 new file mode 100644 index 00000000..827a51c0 --- /dev/null +++ b/config/system_test/metric_report_server.json5 @@ -0,0 +1,10 @@ +{ + "server": "", + "user": "", + "password": "", + "home": "", + "metric_reporter_location": "", + "pythonpath": "", + "target_system_user": "", + "target_system_password": "" +} \ No newline at end of file diff --git a/keywords/system_test/metric_reporter_keywords.py b/keywords/system_test/metric_reporter_keywords.py new file mode 100644 index 00000000..07206c38 --- /dev/null +++ b/keywords/system_test/metric_reporter_keywords.py @@ -0,0 +1,173 @@ +import json +import os +import time +from datetime import datetime + +from config.configuration_manager import ConfigurationManager +from framework.logging.automation_logger import get_logger +from framework.resources.resource_finder import get_stx_resource_path +from framework.ssh.ssh_connection import SSHConnection +from framework.ssh.ssh_connection_manager import SSHConnectionManager +from keywords.base_keyword import BaseKeyword +from keywords.files.file_keywords import FileKeywords + + +METRIC_REPORTER_CONFIG_TEMPLATE = { + "cluster": { + "host": "{host_url}", + "distributedcloud": "no", + "metric_url": "/mon-elasticsearch-client/metricbeat-*/_search?scroll=", + "scroll_url": "/mon-elasticsearch-client/_search/scroll", + "username": "{target_system_user}", + "password": "{target_system_password}" + }, + "dc": { + "ip_addr": "not a dc system", + "ssh_port": "", + "ssh_pass": "not a dc system" + }, + "filters": { + "__SYSNAME__": "{lab_name}", + "__START_TS__": "{start_time}", + "__END_TS__": "{end_time}", + "__TIMEZONE__": "UTC", + "__QSIZE__": "4096", + "__SCROLL__": "15m" + }, + "datasets": [ + {"name": "system.core", "query": "queries/scrolling_dataset.json", "csv": "true"}, + {"name": "system.cpu", "query": "queries/scrolling_dataset.json", "csv": "true"}, + {"name": "linux.iostat", "query": "queries/scrolling_dataset.json", "csv": "true"}, + {"name": "system.diskio", "query": "queries/scrolling_dataset.json", "csv": "true"}, + {"name": "system.filesystem", "query": "queries/scrolling_dataset.json", "csv": "true", "exclude": ["^/var/lib/kubelet/plugins/kubernetes.io/rbd/mounts"]}, + {"name": "system.load", "query": "queries/scrolling_dataset.json", "csv": "true"}, + {"name": "system.memory", "query": "queries/scrolling_dataset.json", "csv": "true"}, + {"name": "system.network", "query": "queries/scrolling_dataset.json", "csv": "true", "exclude": ["^cali"]}, + {"name": "system.process", "query": "queries/scrolling_dataset.json", "csv": "true", "period": "10"}, + {"name": "system.top_n", "query": "queries/scrolling_dataset.json", "csv": "true", "top_n": "", "hosts": ""}, + {"name": "kubernetes.container", "query": "queries/scrolling_dataset.json", "csv": "true", "period": "10"}, + {"name": "kubernetes.pod", "query": "queries/scrolling_dataset.json", "csv": "true", "period": "10"}, + {"name": "kubernetes.node", "query": "queries/scrolling_dataset.json", "csv": "true", "period": "10"} + ], + "reporting": { + "output_dir": "{result_output_dir}", + "json": "true", + "tabular": "true" + } +} + + +class MetricReporterKeywords(BaseKeyword): + """ + Keywords for running metric reporter to collect system metrics during tests. + """ + + def __init__(self, ssh_connection): + """ + Constructor + + Args: + ssh_connection: SSH connection to the target system + """ + self.ssh_connection = ssh_connection + + def run_metric_reporter(self, start_time: datetime, end_time: datetime) -> str: + """ + Run metric reporter to collect system metrics for the specified time period. + + Args: + start_time (datetime): Start time for metric collection + end_time (datetime): End time for metric collection + + Returns: + str: Local directory path where metrics were downloaded + """ + + get_logger().log_info(f"Loading metric server configuration...") + server_config_path = get_stx_resource_path("config/system_test/metric_report_server.json5") + with open(server_config_path, 'r') as f: + server_config = json.load(f) + + # Extract values from server config + result_output_dir = f"{server_config['home']}/metric_reports_{int(time.time())}" + metric_server = server_config["server"] + metric_user = server_config["user"] + metric_password = server_config["password"] + metric_reporter_location = server_config["metric_reporter_location"] + pythonpath = server_config["pythonpath"] + + # Create SSH connection to metric server + metric_ssh = SSHConnectionManager.create_ssh_connection( + metric_server, + metric_user, + metric_password, + ) + + get_logger().log_debug(f"Creating remote directory: {result_output_dir} and content config file") + metric_ssh.send(f"mkdir -p {result_output_dir}") + self.validate_success_return_code(metric_ssh) + config_content = self._create_config_content(start_time, end_time, result_output_dir, server_config) + config_file_name = f"report_config_{datetime.utcnow().strftime('%y%m%d%H%M')}.json" + remote_config_path = f"{result_output_dir}/{config_file_name}" + create_config_cmd = f'cat > {remote_config_path} << "EOF"\n{config_content}\nEOF' + metric_ssh.send(create_config_cmd) + self.validate_success_return_code(metric_ssh) + get_logger().log_debug(f"Created config file on remote server: {remote_config_path}") + + #set PYTHONPATH and run metric reporter + setup_cmd = f"export PYTHONPATH={pythonpath}" + metric_ssh.send(setup_cmd) + self.validate_success_return_code(metric_ssh) + run_cmd = f"python3 {metric_reporter_location}/main.py -c {remote_config_path}" + get_logger().log_info(f"Running metric reporter: {run_cmd}") + output = metric_ssh.send_as_sudo(run_cmd) + self.validate_success_return_code(metric_ssh) + get_logger().log_info(f"Metric reporter output: {''.join(output)}") + + local_dest = os.path.join(get_logger().get_test_case_log_dir(), os.path.basename(result_output_dir)) + get_logger().log_info(f"Downloading metric results from {result_output_dir} to {local_dest}") + + os.makedirs(local_dest, exist_ok=True) + file_keywords = FileKeywords(metric_ssh) + for filename in file_keywords.get_files_in_dir(result_output_dir): + remote_file = f"{result_output_dir}/{filename}" + local_file = os.path.join(local_dest, filename) + file_keywords.download_file(remote_file, local_file) + get_logger().log_info(f"Metric results downloaded to: {local_dest}") + + return local_dest + + + def _create_config_content(self, start_time: datetime, end_time: datetime, + result_output_dir: str, server_config: dict) -> str: + """ + Create metric reporter configuration content in JSON format using template. + + Args: + start_time (datetime): Start time for metric collection + end_time (datetime): End time for metric collection + result_output_dir (str): Output directory for results + lab_name (str): Lab name + lab_ip (str): Lab IP address + + Returns: + str: JSON configuration content + """ + # Get lab configuration + lab_config = ConfigurationManager.get_lab_config() + lab_name = lab_config.get_lab_name() + lab_ip = lab_config.get_floating_ip() + + # Create config from template + config_str = json.dumps(METRIC_REPORTER_CONFIG_TEMPLATE) + host_url = f"https://[{lab_ip}]:31001" if ":" in lab_ip else f"https://{lab_ip}:31001" + config_str = config_str.replace("{host_url}", host_url) + config_str = config_str.replace("{lab_name}", lab_name) + config_str = config_str.replace("{start_time}", start_time.strftime('%Y-%m-%dT%H:%M:%S.%f')) + config_str = config_str.replace("{end_time}", end_time.strftime('%Y-%m-%dT%H:%M:%S.%f')) + config_str = config_str.replace("{result_output_dir}", result_output_dir) + config_str = config_str.replace("{target_system_user}", server_config.get("target_system_user", "")) + config_str = config_str.replace("{target_system_password}", server_config.get("target_system_password", "")) + config = json.loads(config_str) + + return json.dumps(config, indent=2) \ No newline at end of file diff --git a/testcases/cloud_platform/system_test/test_container_deployment.py b/testcases/cloud_platform/system_test/test_container_deployment.py index c11be0f3..0ab18d88 100644 --- a/testcases/cloud_platform/system_test/test_container_deployment.py +++ b/testcases/cloud_platform/system_test/test_container_deployment.py @@ -10,6 +10,7 @@ from keywords.k8s.namespace.kubectl_delete_namespace_keywords import KubectlDele from keywords.k8s.deployments.kubectl_get_deployments_keywords import KubectlGetDeploymentsKeywords from keywords.system_test.timing_logger import TimingLogger from keywords.system_test.setup_stress_pods import SetupStressPods +from keywords.system_test.metric_reporter_keywords import MetricReporterKeywords @mark.p0 @@ -62,16 +63,31 @@ def deploy_benchmark_pods(request, benchmark): namespace = f"{benchmark}-benchmark" timing_logger = TimingLogger(f"{benchmark}_container_deployment") + start_time = datetime.utcnow() stress_pods = SetupStressPods(ssh_connection) deploy_time, scale_up_time = stress_pods.setup_stress_pods(benchmark=benchmark) get_logger().log_test_case_step("Scaling down all deployments and calculating time...") scale_down_time = stress_pods.scale_deployments(0, namespace) get_logger().log_info(f"Time to scale down pods: {scale_down_time:.2f} seconds") + end_time = datetime.utcnow() - # Log all timings to CSV and HTML files + get_logger().log_test_case_step("Logging timings for deployment operations...") timing_logger.log_timings(deploy_time, scale_up_time, scale_down_time) + get_logger().log_test_case_step("Running metric reporter to collect system metrics...") + metric_reporter = MetricReporterKeywords(ssh_connection) + local_metrics_dir = metric_reporter.run_metric_reporter(start_time, end_time) + + get_logger().log_test_case_step(f"Downloading collectd logs to {local_metrics_dir}") + sftp_client = ssh_connection.get_sftp_client() + for filename in sftp_client.listdir("/var/log"): + if filename.startswith("collectd"): + remote_file = f"/var/log/{filename}" + local_file = f"{local_metrics_dir}/{filename}" + sftp_client.get(remote_file, local_file) + get_logger().log_info(f"Downloaded {filename}") + def teardown(): deployments_output = KubectlGetDeploymentsKeywords(ssh_connection).get_deployments(namespace=namespace) deployments_objs = deployments_output.get_deployments()