Merge "Adding metric reporter keyword for running metric reporter tool."

config/system_test/metric_report_server.json5

@@ -0,0 +1,10 @@
+{
+  "server": "",
+  "user": "",
+  "password": "",
+  "home": "",
+  "metric_reporter_location": "",
+  "pythonpath": "",
+  "target_system_user": "",
+  "target_system_password": ""
+}

keywords/system_test/metric_reporter_keywords.py

@@ -0,0 +1,173 @@
+import json
+import os
+import time
+from datetime import datetime
+
+from config.configuration_manager import ConfigurationManager
+from framework.logging.automation_logger import get_logger
+from framework.resources.resource_finder import get_stx_resource_path
+from framework.ssh.ssh_connection import SSHConnection
+from framework.ssh.ssh_connection_manager import SSHConnectionManager
+from keywords.base_keyword import BaseKeyword
+from keywords.files.file_keywords import FileKeywords
+
+
+METRIC_REPORTER_CONFIG_TEMPLATE = {
+    "cluster": {
+        "host": "{host_url}",
+        "distributedcloud": "no",
+        "metric_url": "/mon-elasticsearch-client/metricbeat-*/_search?scroll=",
+        "scroll_url": "/mon-elasticsearch-client/_search/scroll",
+        "username": "{target_system_user}",
+        "password": "{target_system_password}"
+    },
+    "dc": {
+        "ip_addr": "not a dc system",
+        "ssh_port": "",
+        "ssh_pass": "not a dc system"
+    },
+    "filters": {
+        "__SYSNAME__": "{lab_name}",
+        "__START_TS__": "{start_time}",
+        "__END_TS__": "{end_time}",
+        "__TIMEZONE__": "UTC",
+        "__QSIZE__": "4096",
+        "__SCROLL__": "15m"
+    },
+    "datasets": [
+        {"name": "system.core", "query": "queries/scrolling_dataset.json", "csv": "true"},
+        {"name": "system.cpu", "query": "queries/scrolling_dataset.json", "csv": "true"},
+        {"name": "linux.iostat", "query": "queries/scrolling_dataset.json", "csv": "true"},
+        {"name": "system.diskio", "query": "queries/scrolling_dataset.json", "csv": "true"},
+        {"name": "system.filesystem", "query": "queries/scrolling_dataset.json", "csv": "true", "exclude": ["^/var/lib/kubelet/plugins/kubernetes.io/rbd/mounts"]},
+        {"name": "system.load", "query": "queries/scrolling_dataset.json", "csv": "true"},
+        {"name": "system.memory", "query": "queries/scrolling_dataset.json", "csv": "true"},
+        {"name": "system.network", "query": "queries/scrolling_dataset.json", "csv": "true", "exclude": ["^cali"]},
+        {"name": "system.process", "query": "queries/scrolling_dataset.json", "csv": "true", "period": "10"},
+        {"name": "system.top_n", "query": "queries/scrolling_dataset.json", "csv": "true", "top_n": "", "hosts": ""},
+        {"name": "kubernetes.container", "query": "queries/scrolling_dataset.json", "csv": "true", "period": "10"},
+        {"name": "kubernetes.pod", "query": "queries/scrolling_dataset.json", "csv": "true", "period": "10"},
+        {"name": "kubernetes.node", "query": "queries/scrolling_dataset.json", "csv": "true", "period": "10"}
+    ],
+    "reporting": {
+        "output_dir": "{result_output_dir}",
+        "json": "true",
+        "tabular": "true"
+    }
+}
+
+
+class MetricReporterKeywords(BaseKeyword):
+    """
+    Keywords for running metric reporter to collect system metrics during tests.
+    """
+
+    def __init__(self, ssh_connection):
+        """
+        Constructor
+
+        Args:
+            ssh_connection: SSH connection to the target system
+        """
+        self.ssh_connection = ssh_connection
+
+    def run_metric_reporter(self, start_time: datetime, end_time: datetime) -> str:
+        """
+        Run metric reporter to collect system metrics for the specified time period.
+
+        Args:
+            start_time (datetime): Start time for metric collection
+            end_time (datetime): End time for metric collection
+
+        Returns:
+            str: Local directory path where metrics were downloaded
+        """
+
+        get_logger().log_info(f"Loading metric server configuration...")
+        server_config_path = get_stx_resource_path("config/system_test/metric_report_server.json5")
+        with open(server_config_path, 'r') as f:
+            server_config = json.load(f)
+
+        # Extract values from server config
+        result_output_dir = f"{server_config['home']}/metric_reports_{int(time.time())}"
+        metric_server = server_config["server"]
+        metric_user = server_config["user"]
+        metric_password = server_config["password"]
+        metric_reporter_location = server_config["metric_reporter_location"]
+        pythonpath = server_config["pythonpath"]
+
+        # Create SSH connection to metric server
+        metric_ssh = SSHConnectionManager.create_ssh_connection(
+            metric_server,
+            metric_user,
+            metric_password,
+        )
+
+        get_logger().log_debug(f"Creating remote directory: {result_output_dir} and content config file")
+        metric_ssh.send(f"mkdir -p {result_output_dir}")
+        self.validate_success_return_code(metric_ssh)
+        config_content = self._create_config_content(start_time, end_time, result_output_dir, server_config)
+        config_file_name = f"report_config_{datetime.utcnow().strftime('%y%m%d%H%M')}.json"
+        remote_config_path = f"{result_output_dir}/{config_file_name}"
+        create_config_cmd = f'cat > {remote_config_path} << "EOF"\n{config_content}\nEOF'
+        metric_ssh.send(create_config_cmd)
+        self.validate_success_return_code(metric_ssh)
+        get_logger().log_debug(f"Created config file on remote server: {remote_config_path}")
+
+        #set PYTHONPATH and run metric reporter
+        setup_cmd = f"export PYTHONPATH={pythonpath}"
+        metric_ssh.send(setup_cmd)
+        self.validate_success_return_code(metric_ssh)
+        run_cmd = f"python3 {metric_reporter_location}/main.py -c {remote_config_path}"
+        get_logger().log_info(f"Running metric reporter: {run_cmd}")
+        output = metric_ssh.send_as_sudo(run_cmd)
+        self.validate_success_return_code(metric_ssh)
+        get_logger().log_info(f"Metric reporter output: {''.join(output)}")
+
+        local_dest = os.path.join(get_logger().get_test_case_log_dir(), os.path.basename(result_output_dir))
+        get_logger().log_info(f"Downloading metric results from {result_output_dir} to {local_dest}")
+
+        os.makedirs(local_dest, exist_ok=True)
+        file_keywords = FileKeywords(metric_ssh)
+        for filename in file_keywords.get_files_in_dir(result_output_dir):
+            remote_file = f"{result_output_dir}/{filename}"
+            local_file = os.path.join(local_dest, filename)
+            file_keywords.download_file(remote_file, local_file)
+        get_logger().log_info(f"Metric results downloaded to: {local_dest}")
+
+        return local_dest
+
+
+    def _create_config_content(self, start_time: datetime, end_time: datetime, 
+                              result_output_dir: str, server_config: dict) -> str:
+        """
+        Create metric reporter configuration content in JSON format using template.
+
+        Args:
+            start_time (datetime): Start time for metric collection
+            end_time (datetime): End time for metric collection
+            result_output_dir (str): Output directory for results
+            lab_name (str): Lab name
+            lab_ip (str): Lab IP address
+
+        Returns:
+            str: JSON configuration content
+        """
+        # Get lab configuration
+        lab_config = ConfigurationManager.get_lab_config()
+        lab_name = lab_config.get_lab_name()
+        lab_ip = lab_config.get_floating_ip()
+
+        # Create config from template
+        config_str = json.dumps(METRIC_REPORTER_CONFIG_TEMPLATE)
+        host_url = f"https://[{lab_ip}]:31001" if ":" in lab_ip else f"https://{lab_ip}:31001"
+        config_str = config_str.replace("{host_url}", host_url)
+        config_str = config_str.replace("{lab_name}", lab_name)
+        config_str = config_str.replace("{start_time}", start_time.strftime('%Y-%m-%dT%H:%M:%S.%f'))
+        config_str = config_str.replace("{end_time}", end_time.strftime('%Y-%m-%dT%H:%M:%S.%f'))
+        config_str = config_str.replace("{result_output_dir}", result_output_dir)
+        config_str = config_str.replace("{target_system_user}", server_config.get("target_system_user", ""))
+        config_str = config_str.replace("{target_system_password}", server_config.get("target_system_password", ""))      
+        config = json.loads(config_str)
+
+        return json.dumps(config, indent=2)

testcases/cloud_platform/system_test/test_container_deployment.py

@@ -10,6 +10,7 @@ from keywords.k8s.namespace.kubectl_delete_namespace_keywords import KubectlDele
 from keywords.k8s.deployments.kubectl_get_deployments_keywords import KubectlGetDeploymentsKeywords
 from keywords.system_test.timing_logger import TimingLogger
 from keywords.system_test.setup_stress_pods import SetupStressPods
+from keywords.system_test.metric_reporter_keywords import MetricReporterKeywords
 
 
 @mark.p0
@@ -62,16 +63,31 @@ def deploy_benchmark_pods(request, benchmark):
     namespace = f"{benchmark}-benchmark"
     timing_logger = TimingLogger(f"{benchmark}_container_deployment")
 
+    start_time = datetime.utcnow()
     stress_pods = SetupStressPods(ssh_connection)
     deploy_time, scale_up_time = stress_pods.setup_stress_pods(benchmark=benchmark)
 
     get_logger().log_test_case_step("Scaling down all deployments and calculating time...")
     scale_down_time = stress_pods.scale_deployments(0, namespace)
     get_logger().log_info(f"Time to scale down pods: {scale_down_time:.2f} seconds")
+    end_time = datetime.utcnow()
 
-    # Log all timings to CSV and HTML files
+    get_logger().log_test_case_step("Logging timings for deployment operations...")
     timing_logger.log_timings(deploy_time, scale_up_time, scale_down_time)
 
+    get_logger().log_test_case_step("Running metric reporter to collect system metrics...")
+    metric_reporter = MetricReporterKeywords(ssh_connection)
+    local_metrics_dir = metric_reporter.run_metric_reporter(start_time, end_time)
+
+    get_logger().log_test_case_step(f"Downloading collectd logs to {local_metrics_dir}")
+    sftp_client = ssh_connection.get_sftp_client()
+    for filename in sftp_client.listdir("/var/log"):
+        if filename.startswith("collectd"):
+            remote_file = f"/var/log/{filename}"
+            local_file = f"{local_metrics_dir}/{filename}"
+            sftp_client.get(remote_file, local_file)
+            get_logger().log_info(f"Downloaded {filename}")
+
     def teardown():
         deployments_output = KubectlGetDeploymentsKeywords(ssh_connection).get_deployments(namespace=namespace)
         deployments_objs = deployments_output.get_deployments()