common/sender: Deal with ReadTimeout in send()
If there's a ReadTimeout when sending a HTTP request, the plugin now retries before exiting. Change-Id: I0218bd412419079ec9544810a6ab863f3de1a915
This commit is contained in:
parent
f004cdc6d2
commit
02b65dc416
|
@ -21,6 +21,7 @@ import threading
|
|||
|
||||
import requests
|
||||
import six
|
||||
import time
|
||||
|
||||
from collectd_ceilometer.common.keystone_light import ClientV3
|
||||
from collectd_ceilometer.common.keystone_light import KeystoneException
|
||||
|
@ -123,8 +124,12 @@ class Sender(object):
|
|||
def _handle_http_error(self, exc, metername, payload, auth_token):
|
||||
raise exc
|
||||
|
||||
def send(self, metername, payload, **kwargs):
|
||||
"""Send the payload to Gnocchi/Aodh"""
|
||||
def send(self, metername, payload, retry=0, **kwargs):
|
||||
"""Send the payload to Gnocchi/Aodh
|
||||
|
||||
:param retry: The number of times to attempt sending when the request
|
||||
times out. Default is 0, which does not retry.
|
||||
"""
|
||||
|
||||
# get the auth_token
|
||||
auth_token = self._authenticate()
|
||||
|
@ -174,6 +179,15 @@ class Sender(object):
|
|||
self._handle_http_error(exc, metername, payload,
|
||||
auth_token, **kwargs)
|
||||
|
||||
except requests.exceptions.ReadTimeout as rto:
|
||||
if retry > 0:
|
||||
LOGGER.debug("ReadTimeout Error.. trying again...")
|
||||
time.sleep(1)
|
||||
self.send(metername, payload, retry=(retry - 1), **kwargs)
|
||||
else:
|
||||
LOGGER.error("Too many timeouts trying to send\n", rto)
|
||||
raise rto
|
||||
|
||||
@classmethod
|
||||
def _perform_request(cls, url, payload, auth_token, req_type="post"):
|
||||
"""Perform the POST/PUT request."""
|
||||
|
|
|
@ -31,6 +31,93 @@ class TestSender(unittest.TestCase):
|
|||
super(TestSender, self).setUp()
|
||||
self.sender = common_sender.Sender()
|
||||
|
||||
self.sender._url_base = \
|
||||
"http://my-gnocchi-endpoint/v1/action"
|
||||
|
||||
@mock.patch('time.sleep')
|
||||
@mock.patch.object(common_sender.Sender, '_create_request_url')
|
||||
@mock.patch.object(common_sender, 'LOGGER')
|
||||
@mock.patch.object(common_sender.Sender, '_perform_request')
|
||||
def test_send_readtimeout_error(self, sender_perf_req,
|
||||
logger, sender_create_request_url,
|
||||
time_sleep):
|
||||
"""Tests the behaviour of send when a timeout error occurs.
|
||||
|
||||
Set-up: There is a ReadTimeout side effect from _perform_request
|
||||
Test: Call send
|
||||
Expected behaviour: Send will be called a second time.
|
||||
"""
|
||||
response_created = requests.Response()
|
||||
response_created.status_code = 201
|
||||
|
||||
self.sender._auth_token = "my-auth-token"
|
||||
|
||||
sender_perf_req.side_effect = [requests.exceptions.ReadTimeout,
|
||||
response_created]
|
||||
|
||||
self.sender.send(metername="my-metername",
|
||||
payload="my-payload",
|
||||
retry=1
|
||||
)
|
||||
|
||||
logger.debug.assert_called_with("ReadTimeout Error.. trying again...")
|
||||
|
||||
@mock.patch('time.sleep')
|
||||
@mock.patch.object(common_sender.Sender, '_create_request_url')
|
||||
@mock.patch.object(common_sender, 'LOGGER')
|
||||
@mock.patch.object(common_sender.Sender, '_perform_request')
|
||||
def test_send_readtimeout_error_retry_0(self, sender_perf_req, logger,
|
||||
sender_create_request_url,
|
||||
time_sleep):
|
||||
"""Tests that ReadTimeout error is raised when retry=0.
|
||||
|
||||
Set-up: There is a ReadTimeout side effect from _perform_request.
|
||||
Test: Call send(.... retry=0)
|
||||
Expected behaviour:
|
||||
* ReadTimeout is raised from send()
|
||||
* there are no re-send attempts (perform_request is only called once)
|
||||
"""
|
||||
self.sender._auth_token = "my-auth-token"
|
||||
|
||||
sender_perf_req.side_effect = requests.exceptions.ReadTimeout
|
||||
|
||||
with self.assertRaises(requests.exceptions.ReadTimeout):
|
||||
self.sender.send(metername="my-metername",
|
||||
payload="my-payload",
|
||||
retry=0
|
||||
)
|
||||
logger.debug.assert_not_called()
|
||||
sender_perf_req.called_once()
|
||||
logger.error.assert_called_with("Too many timeouts trying to send\n",
|
||||
mock.ANY)
|
||||
|
||||
@mock.patch('time.sleep')
|
||||
@mock.patch.object(common_sender.Sender, '_create_request_url')
|
||||
@mock.patch.object(common_sender, 'LOGGER')
|
||||
@mock.patch.object(common_sender.Sender, '_perform_request')
|
||||
def test_send_timeout_recurrent(self, sender_perf_req, logger,
|
||||
sender_create_request_url,
|
||||
time_sleep):
|
||||
"""Test that resending is attempted when there's a ReadTimout error.
|
||||
|
||||
Set-up:
|
||||
Test: Call send() with retry=<some_number>
|
||||
Expected behaviour:
|
||||
* TimeoutError will be re-raised after attempting multiple times
|
||||
"""
|
||||
self.sender._auth_token = "my-auth-token"
|
||||
sender_perf_req.side_effect = requests.exceptions.ReadTimeout
|
||||
|
||||
with self.assertRaises(requests.exceptions.ReadTimeout):
|
||||
self.sender.send(metername="my-metername",
|
||||
payload="my-payload",
|
||||
retry=5
|
||||
)
|
||||
|
||||
logger.debug.assert_called_with("ReadTimeout Error.. trying again...")
|
||||
sender_perf_req.assert_called()
|
||||
time_sleep.assert_has_calls([mock.call(1)] * 5)
|
||||
|
||||
@mock.patch.object(requests, 'post')
|
||||
@mock.patch.object(requests, 'get')
|
||||
def test_perform_request_req_type_get(self, get, post):
|
||||
|
|
Loading…
Reference in New Issue