From a21cb2a0af5be96eb8cebb8445d03ed2c1cb110a Mon Sep 17 00:00:00 2001 From: "Chris Straut (cs4987)" Date: Thu, 26 Aug 2021 10:38:31 -0500 Subject: [PATCH] Fix issue with retry logic querying prometheus The issue is that the successful response from prometheus wasn't triggering of the exit from the retry loop. Now on successful queries the while retry loop will break into a successful exit strategy. Change-Id: I528c1c17d2131256097cac5a67ec7ea17541c685 --- nagios/plugins/check_exporter_health_metric.py | 1 + nagios/plugins/query_prometheus_alerts.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/nagios/plugins/check_exporter_health_metric.py b/nagios/plugins/check_exporter_health_metric.py index df23161c..92bebf0d 100755 --- a/nagios/plugins/check_exporter_health_metric.py +++ b/nagios/plugins/check_exporter_health_metric.py @@ -113,6 +113,7 @@ def query_exporter_metric(exporter_namespace, label_selector, metric_name): for metric in line_item_metrics: metric_with_labels, value = metric.split(" ") metrics[metric_with_labels] = float(value) + break except Exception as e: if retry < max_retry: print('Request timeout, Retrying - {}'.format(retry)) diff --git a/nagios/plugins/query_prometheus_alerts.py b/nagios/plugins/query_prometheus_alerts.py index 542f83a6..73fbe57e 100755 --- a/nagios/plugins/query_prometheus_alerts.py +++ b/nagios/plugins/query_prometheus_alerts.py @@ -154,6 +154,7 @@ def query_prometheus(prometheus_api, alertname, labels_csv, timeout): response = requests.get(include_schema( prometheus_api) + "/api/v1/query", **kwargs) response_json = response.json() + break except requests.exceptions.Timeout: if retry < max_retry: print('Request timeout, Retrying - {}'.format(retry)) @@ -214,6 +215,7 @@ def check_prom_metrics_available(prometheus_api, metrics, labels_csv, timeout): metrics_available = False else: metrics_available = True + break except requests.exceptions.Timeout: if retry < max_retry: retry += 1