From d3506532fd08484d5d63f5875fcd3455a52aa6c8 Mon Sep 17 00:00:00 2001 From: Daniel Pawlik Date: Mon, 9 May 2022 08:54:38 +0200 Subject: [PATCH] Handle exception: TOO_MANY_REQUESTS The Opensearch cluster might be sometimes overloaded, so the logsender receive an exception: opensearchpy.exceptions.TransportError: TransportError(429, 'cluster_block_exception', 'index [logstash-logs-2022.05.07] blocked by: [TOO_MANY_REQUESTS/12/disk usage exceeded flood-stage watermark, index has read-only-allow-delete block];') In that case, logsender will sleep 120 seconds before continue. Change-Id: I4973fbb5a38cc528ed750378cff068def62c2dd0 --- logscraper/logsender.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/logscraper/logsender.py b/logscraper/logsender.py index c599c53..74ac8e3 100755 --- a/logscraper/logsender.py +++ b/logscraper/logsender.py @@ -384,6 +384,16 @@ def create_indices(index, args): if e.error.lower() == 'resource_already_exists_exception': logging.debug("The indices already exists, continue") return True + except opensearch_exceptions.TransportError as e: + # NOTE(dpawlik) To avoid error: "TOO_MANY_REQUESTS/12/disk usage + # exceeded flood-stage watermark", let's wait some time before + # continue. + if 'too_many_requests' in e.error.lower(): + logging.warning("Cluster is probably overloaded/flooded. " + "Logsender will wait some time, then continue." + "Exception details: %s" % e) + time.sleep(120) + return True def prepare_and_send(ready_directories, args):