Parcourir la source

Merge "Timeout connections when watching K8s API"

tags/1.1.0
Zuul il y a 2 mois
Parent
révision
039f2aaa3d

+ 12
- 0
kuryr_kubernetes/config.py Voir le fichier

@@ -155,6 +155,18 @@ k8s_opts = [
cfg.IntOpt('watch_retry_timeout',
help=_('Time (in seconds) the watcher retries watching for.'),
default=60),
cfg.IntOpt('watch_connection_timeout',
help=_('TCP connection timeout (in seconds) for the watcher '
'connections to K8s API.'),
default=30),
cfg.IntOpt('watch_read_timeout',
help=_('TCP read timeout (in seconds) for the watcher '
'connections to K8s API. This affects reaction to time '
'when there are no events being streamed from K8s API. '
'When too low, Kuryr will reconnect more often. When '
'too high, Kuryr will take longer to reconnect when K8s '
'API stream was being silently broken.'),
default=60),
cfg.ListOpt('enabled_handlers',
help=_("The comma-separated handlers that should be "
"registered for watching in the pipeline."),

+ 32
- 12
kuryr_kubernetes/k8s_client.py Voir le fichier

@@ -15,6 +15,7 @@
import contextlib
import itertools
import os
import ssl

from oslo_log import log as logging
from oslo_serialization import jsonutils
@@ -25,6 +26,7 @@ from kuryr_kubernetes import config
from kuryr_kubernetes import constants
from kuryr_kubernetes import exceptions as exc

CONF = config.CONF
LOG = logging.getLogger(__name__)


@@ -240,21 +242,39 @@ class K8sClient(object):
raise exc.K8sClientException(response.text)

def watch(self, path):
params = {'watch': 'true'}
url = self._base_url + path
resource_version = None
header = {}
timeouts = (CONF.kubernetes.watch_connection_timeout,
CONF.kubernetes.watch_read_timeout)
if self.token:
header.update({'Authorization': 'Bearer %s' % self.token})

# TODO(ivc): handle connection errors and retry on failure
while True:
with contextlib.closing(
requests.get(url, params=params, stream=True,
cert=self.cert, verify=self.verify_server,
headers=header)) as response:
if not response.ok:
raise exc.K8sClientException(response.text)
for line in response.iter_lines():
line = line.decode('utf-8').strip()
if line:
yield jsonutils.loads(line)
try:
params = {'watch': 'true'}
if resource_version:
params['resourceVersion'] = resource_version
with contextlib.closing(
requests.get(
url, params=params, stream=True, cert=self.cert,
verify=self.verify_server, headers=header,
timeout=timeouts)) as response:
if not response.ok:
raise exc.K8sClientException(response.text)
for line in response.iter_lines():
line = line.decode('utf-8').strip()
if line:
line_dict = jsonutils.loads(line)
yield line_dict
# Saving the resourceVersion in case of a restart.
# At this point it's safely passed to handler.
m = line_dict.get('object', {}).get('metadata', {})
resource_version = m.get('resourceVersion', None)
except (requests.ReadTimeout, ssl.SSLError) as e:
if isinstance(e, ssl.SSLError) and e.args != ('timed out',):
raise

LOG.warning('%ds without data received from watching %s. '
'Retrying the connection with resourceVersion=%s.',
timeouts[1], path, params.get('resourceVersion'))

+ 28
- 1
kuryr_kubernetes/tests/unit/test_k8s_client.py Voir le fichier

@@ -335,7 +335,34 @@ class TestK8sClient(test_base.TestCase):
self.assertEqual(cycles, m_resp.close.call_count)
m_get.assert_called_with(self.base_url + path, headers={}, stream=True,
params={'watch': 'true'}, cert=(None, None),
verify=False)
verify=False, timeout=(30, 60))

@mock.patch('requests.get')
def test_watch_restart(self, m_get):
path = '/test'
data = [{'object': {'metadata': {'name': 'obj%s' % i,
'resourceVersion': i}}}
for i in range(3)]
lines = [jsonutils.dump_as_bytes(i) for i in data]

m_resp = mock.MagicMock()
m_resp.ok = True
m_resp.iter_lines.side_effect = [lines, requests.ReadTimeout, lines]
m_get.return_value = m_resp

self.assertEqual(data * 2,
list(itertools.islice(self.client.watch(path),
len(data) * 2)))
self.assertEqual(3, m_get.call_count)
self.assertEqual(3, m_resp.close.call_count)
m_get.assert_any_call(
self.base_url + path, headers={}, stream=True,
params={"watch": "true"}, cert=(None, None), verify=False,
timeout=(30, 60))
m_get.assert_any_call(
self.base_url + path, headers={}, stream=True,
params={"watch": "true", "resourceVersion": 2}, cert=(None, None),
verify=False, timeout=(30, 60))

@mock.patch('requests.get')
def test_watch_exception(self, m_get):

Chargement…
Annuler
Enregistrer