Browse Source

Make Request Timeout Configurable

As the size of the YAMLs increases, the amount of time needed
to process the request increased as well. Hence there is a need
to make 'timeout' configurable for the deckhand client.

Change-Id: Iab91091cd8b9a900ad0daeac22e435d4e5c9c97d
changes/87/569187/1
Anthony Lin 5 years ago
parent
commit
25236ac89b
  1. 5
      charts/shipyard/values.yaml
  2. 15
      etc/shipyard/shipyard.conf.sample
  3. 37
      shipyard_airflow/conf/config.py
  4. 12
      shipyard_airflow/control/configdocs/configdocs_helper.py
  5. 48
      shipyard_airflow/control/configdocs/deckhand_client.py
  6. 5
      tests/unit/control/test.conf
  7. 5
      tools/resources/shipyard.conf

5
charts/shipyard/values.yaml

@ -334,6 +334,11 @@ conf:
auth_section: keystone_authtoken
auth_version: v3
memcache_security_strategy: ENCRYPT
requests_config:
deckhand_client_connect_timeout: 5
deckhand_client_read_timeout: 300
validation_connect_timeout: 5
validation_read_timeout: 300
airflow:
override:
append:

15
etc/shipyard/shipyard.conf.sample

@ -277,6 +277,21 @@
#log_level = 10
[requests_config]
# Deckhand client connect timeout (in seconds)
#deckhand_client_connect_timeout = 5
# Deckhand client timeout (in seconds) for GET,
# PUT, POST and DELETE request
#deckhand_client_read_timeout = 300
# UCP component validation connect timeout (in seconds)
#validation_connect_timeout = 5
# UCP component validation timeout (in seconds)
#validation_read_timeout = 300
[shipyard]
#

37
shipyard_airflow/conf/config.py

@ -77,7 +77,7 @@ SECTIONS = [
help=(
'The service type for the service playing the role '
'of Shipyard. The specified type is used to perform '
'the service lookup in the Keystone service catalog. '
'the service lookup in the Keystone service catalog.'
)
),
]
@ -92,7 +92,7 @@ SECTIONS = [
help=(
'The service type for the service playing the role '
'of Deckhand. The specified type is used to perform '
'the service lookup in the Keystone service catalog. '
'the service lookup in the Keystone service catalog.'
)
),
]
@ -107,7 +107,7 @@ SECTIONS = [
help=(
'The service type for the service playing the role '
'of Armada. The specified type is used to perform '
'the service lookup in the Keystone service catalog. '
'the service lookup in the Keystone service catalog.'
)
),
]
@ -122,7 +122,7 @@ SECTIONS = [
help=(
'The service type for the service playing the role '
'of Drydock. The specified type is used to perform '
'the service lookup in the Keystone service catalog. '
'the service lookup in the Keystone service catalog.'
)
),
cfg.IntOpt(
@ -182,6 +182,35 @@ SECTIONS = [
),
]
),
ConfigSection(
name='requests_config',
title='Requests Configuration',
options=[
cfg.IntOpt(
'deckhand_client_connect_timeout',
default=5,
help='Deckhand client connect timeout (in seconds)'
),
cfg.IntOpt(
'deckhand_client_read_timeout',
default=300,
help=(
'Deckhand client timeout (in seconds) for GET, '
'PUT, POST and DELETE request'
)
),
cfg.IntOpt(
'validation_connect_timeout',
default=5,
help='UCP component validation connect timeout (in seconds)'
),
cfg.IntOpt(
'validation_read_timeout',
default=300,
help='UCP component validation timeout (in seconds)'
),
]
),
]

12
shipyard_airflow/control/configdocs/configdocs_helper.py

@ -437,13 +437,13 @@ class ConfigdocsHelper(object):
'content-type': 'application/json'
}
# TODO: We will need to make timeout a configurable value as it
# will differ from site to site based on the size of the rendered
# document
# Note that 30 seconds is not sufficient to complete validations.
# Hence we are increaing the default timeout to 60 seconds.
http_resp = requests.post(
url, headers=headers, data=design_reference, timeout=(5, 60))
url,
headers=headers,
data=design_reference,
timeout=(
CONF.requests_config.validation_connect_timeout,
CONF.requests_config.validation_read_timeout))
# 400 response is "valid" failure to validate. > 400 is a problem.
if http_resp.status_code > 400:
http_resp.raise_for_status()

48
shipyard_airflow/control/configdocs/deckhand_client.py

@ -362,11 +362,14 @@ class DeckhandClient(object):
headers['content-type'] = 'application/x-yaml'
DeckhandClient._log_request('PUT', url, params)
response = requests.put(url,
params=params,
headers=headers,
data=document_data,
timeout=(5, 30))
response = requests.put(
url,
params=params,
headers=headers,
data=document_data,
timeout=(
CONF.requests_config.deckhand_client_connect_timeout,
CONF.requests_config.deckhand_client_read_timeout))
return response
except RequestException as rex:
LOG.error(rex)
@ -386,10 +389,13 @@ class DeckhandClient(object):
}
DeckhandClient._log_request('GET', url, params)
response = requests.get(url,
params=params,
headers=headers,
timeout=(5, 30))
response = requests.get(
url,
params=params,
headers=headers,
timeout=(
CONF.requests_config.deckhand_client_connect_timeout,
CONF.requests_config.deckhand_client_read_timeout))
return response
except RequestException as rex:
LOG.error(rex)
@ -411,11 +417,14 @@ class DeckhandClient(object):
headers['content-type'] = 'application/x-yaml'
DeckhandClient._log_request('POST', url, params)
response = requests.post(url,
params=params,
headers=headers,
data=document_data,
timeout=(5, 30))
response = requests.post(
url,
params=params,
headers=headers,
data=document_data,
timeout=(
CONF.requests_config.deckhand_client_connect_timeout,
CONF.requests_config.deckhand_client_read_timeout))
return response
except RequestException as rex:
LOG.error(rex)
@ -434,10 +443,13 @@ class DeckhandClient(object):
}
DeckhandClient._log_request('DELETE', url, params)
response = requests.delete(url,
params=params,
headers=headers,
timeout=(5, 30))
response = requests.delete(
url,
params=params,
headers=headers,
timeout=(
CONF.requests_config.deckhand_client_connect_timeout,
CONF.requests_config.deckhand_client_read_timeout))
return response
except RequestException as rex:
LOG.error(rex)

5
tests/unit/control/test.conf

@ -35,6 +35,11 @@ project_domain_name = default
project_name = service
user_domain_name = default
username = shipyard
[requests_config]
deckhand_client_connect_timeout=5
deckhand_client_read_timeout=300
validation_connect_timeout=5
validation_read_timeout=300
[shipyard]
service_type = shipyard

5
tools/resources/shipyard.conf

@ -37,5 +37,10 @@ project_domain_name = default
project_name = service
user_domain_name = default
username = shipyard
[requests_config]
deckhand_client_connect_timeout=5
deckhand_client_read_timeout=300
validation_connect_timeout=5
validation_read_timeout=300
[shipyard]
service_type = shipyard

Loading…
Cancel
Save