Create a function for deleting old injected metrics on prometheus

Change-Id: I215d748d6a57cbeb0e0976dbce292dd88040fef2
Signed-off-by: morenod <dsanzmor@redhat.com>

watcher_tempest_plugin/services/metric/prometheus_client.py

@@ -15,6 +15,7 @@
 import json
 
 from oslo_log import log
+from urllib.parse import quote
 
 from watcher_tempest_plugin.services import base
 
@@ -62,12 +63,15 @@ class PromtoolClient:
 
         self.prometheus_url = url
         self.prometheus_write_url = url + "/api/v1/write"
+        self.prometheus_delete_series_url = (
+            url + "/api/v1/admin/tsdb/delete_series")
         # active targets
         self.prometheus_targets_url = url + "/api/v1/targets?state=active"
         self.prometheus_fqdn_label = prometheus_fqdn_label
         self.promtool_cmd = [promtool_path]
         if prometheus_ssl_cert:
             self.promtool_cmd.insert(0, f"SSL_CERT_DIR={prometheus_ssl_cert}")
+
         # Map hostnames and fqdn to prometheus instances
         self._prometheus_instances = {}
 
@@ -122,6 +126,27 @@ class PromtoolClient:
 
         return self.client.exec_cmd(cmd)
 
+    def delete_series(self, expr='{job="promtool"}'):
+        """Delete prometheus time series with the specified parameters.
+
+        :param expr: expression to match the time series
+        :returns: Stdout output generated by the command.
+        """
+
+        # Having a lot of problems passing the expression
+        # with double quotes both on direct command and
+        # ssh command so using url encoding
+
+        quoted_expr = quote(expr)
+
+        cmd = ['curl', '-s', '-k', '-X', 'POST', '-g',
+               self.prometheus_delete_series_url + '?match[]=' + quoted_expr]
+
+        LOG.debug(f"curl command: {' '.join(cmd)}")
+        out = self.client.exec_cmd(cmd)
+        if len(out) > 1:
+            LOG.debug(f"delete_series execution output: {out}")
+
     def add_measures(self, input_data):
         """Add measures resources with the specified parameters.
 

watcher_tempest_plugin/tests/scenario/base.py

@@ -416,6 +416,20 @@ class BaseInfraOptimScenarioTest(manager.ScenarioTest):
 
         return measures_body
 
+    def clean_injected_metrics(self):
+        """Delete all injected metrics from datastore.
+
+        This is useful to ensure that the tests are not affected by
+        previously injected metrics.
+        """
+        LOG.debug("Deleting injected metrics from Datastore")
+        if CONF.optimize.datasource == "gnocchi":
+            # TODO(morenod): Add function for deleting injected metrics
+            # from Gnocchi.
+            pass
+        elif CONF.optimize.datasource == "prometheus":
+            self.prometheus_client.delete_series()
+
     def make_host_statistic(self, metrics=dict(), loaded_hosts=[]):
         """Add host metrics to the datasource
 

watcher_tempest_plugin/tests/scenario/test_execute_actuator.py

@@ -84,6 +84,7 @@ class TestExecuteActionsViaActuator(base.BaseInfraOptimScenarioTest):
 
     def _prerequisite_param_for_migrate_action(self):
         # This test requires metrics injection
+        self.addCleanup(self.clean_injected_metrics)
         created_instances = self._create_one_instance_per_host()
         source_node = self.get_host_for_server(created_instances[0]['id'])
         destination_node = self.get_host_other_than(created_instances[0]['id'])
@@ -101,6 +102,7 @@ class TestExecuteActionsViaActuator(base.BaseInfraOptimScenarioTest):
 
     def _prerequisite_param_for_resize_action(self):
         # This test requires metrics injection
+        self.addCleanup(self.clean_injected_metrics)
         created_instances = self._create_one_instance_per_host()
         for instance in created_instances:
             self.make_instance_statistic(instance)

watcher_tempest_plugin/tests/scenario/test_execute_basic_optim.py

@@ -71,6 +71,7 @@ class TestExecuteBasicStrategy(base.BaseInfraOptimScenarioTest):
 
         self.addCleanup(self.rollback_compute_nodes_status)
         self.addCleanup(self.wait_delete_instances_from_model)
+        self.addCleanup(self.clean_injected_metrics)
         instances = self._create_one_instance_per_host()
         # wait for compute model updates
         self.wait_for_instances_in_model(instances)

watcher_tempest_plugin/tests/scenario/test_execute_strategies.py

@@ -53,6 +53,7 @@ class TestExecuteStrategies(base.BaseInfraOptimScenarioTest):
     def test_execute_basic_strategy(self):
         self.addCleanup(self.rollback_compute_nodes_status)
         self.addCleanup(self.wait_delete_instances_from_model)
+        self.addCleanup(self.clean_injected_metrics)
         instances = self._create_one_instance_per_host()
         # wait for compute model updates
         self.wait_for_instances_in_model(instances)
@@ -102,6 +103,7 @@ class TestExecuteStrategies(base.BaseInfraOptimScenarioTest):
 
         self.addCleanup(self.rollback_compute_nodes_status)
         self.addCleanup(self.wait_delete_instances_from_model)
+        self.addCleanup(self.clean_injected_metrics)
         metrics = {
             'instance_cpu_usage': {},
             'instance_ram_usage': {},
@@ -131,6 +133,7 @@ class TestExecuteStrategies(base.BaseInfraOptimScenarioTest):
 
         self.addCleanup(self.rollback_compute_nodes_status)
         self.addCleanup(self.wait_delete_instances_from_model)
+        self.addCleanup(self.clean_injected_metrics)
         host = self.get_enabled_compute_nodes()[0]['host']
         instances = []
         for _ in range(2):
@@ -164,6 +167,7 @@ class TestExecuteStrategies(base.BaseInfraOptimScenarioTest):
 
         self.addCleanup(self.rollback_compute_nodes_status)
         self.addCleanup(self.wait_delete_instances_from_model)
+        self.addCleanup(self.clean_injected_metrics)
         host = self.get_enabled_compute_nodes()[0]['host']
         instances = []
         for _ in range(2):

watcher_tempest_plugin/tests/scenario/test_execute_vm_workload_consolidation.py

@@ -71,6 +71,7 @@ class TestExecuteVmWorkloadBalanceStrategy(base.BaseInfraOptimScenarioTest):
 
         self.addCleanup(self.rollback_compute_nodes_status)
         self.addCleanup(self.wait_delete_instances_from_model)
+        self.addCleanup(self.clean_injected_metrics)
         metrics = {
             'instance_cpu_usage': {},
             'instance_ram_usage': {},

watcher_tempest_plugin/tests/scenario/test_execute_workload_balance.py

@@ -57,6 +57,7 @@ class TestExecuteWorkloadBalanceStrategy(base.BaseInfraOptimScenarioTest):
         # This test requires metrics injection
         self.addCleanup(self.rollback_compute_nodes_status)
         self.addCleanup(self.wait_delete_instances_from_model)
+        self.addCleanup(self.clean_injected_metrics)
         host = self.get_enabled_compute_nodes()[0]['host']
         instances = []
         for _ in range(4):
@@ -89,6 +90,7 @@ class TestExecuteWorkloadBalanceStrategy(base.BaseInfraOptimScenarioTest):
         # This test requires metrics injection
         self.addCleanup(self.rollback_compute_nodes_status)
         self.addCleanup(self.wait_delete_instances_from_model)
+        self.addCleanup(self.clean_injected_metrics)
         host = self.get_enabled_compute_nodes()[0]['host']
         instances = []
         for _ in range(4):