From daae073905f3e6afe1f859120156949bc07f572f Mon Sep 17 00:00:00 2001 From: Sergiy Markin Date: Thu, 11 May 2023 19:46:41 +0000 Subject: [PATCH] Shipyard API timeout issue This PS adds code that uses preconfigured drydock-api request timeout and adds these values for promenade as well as using them in the code: drydock_client_connect_timeout: 20 drydock_client_read_timeout: 300 promenade_client_connect_timeout: 20 promenade_client_read_timeout: 300 Change-Id: Ic5b1920257859239613a3ce77134e6b05bd7e9dd --- charts/shipyard/values.yaml | 4 ++++ .../etc/shipyard/shipyard.conf.sample | 8 ++++++++ .../plugins/drydock_validate_design.py | 17 ++++++++++++----- .../plugins/promenade_validate_site_design.py | 17 ++++++++++++----- .../shipyard_client/api_client/base_client.py | 10 +++++++++- 5 files changed, 45 insertions(+), 11 deletions(-) diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml index 3f74f12e..86edd825 100644 --- a/charts/shipyard/values.yaml +++ b/charts/shipyard/values.yaml @@ -413,6 +413,8 @@ conf: auth_version: v3 memcache_security_strategy: ENCRYPT requests_config: + base_client_connect_timeout: 5 + base_client_read_timeout: 300 airflow_log_connect_timeout: 5 airflow_log_read_timeout: 300 deckhand_client_connect_timeout: 5 @@ -423,6 +425,8 @@ conf: notes_read_timeout: 10 drydock_client_connect_timeout: 20 drydock_client_read_timeout: 300 + promenade_client_connect_timeout: 20 + promenade_client_read_timeout: 300 airflow: worker_endpoint_scheme: 'http' worker_port: 8793 diff --git a/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample b/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample index a822dead..317d3591 100644 --- a/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample +++ b/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample @@ -404,6 +404,14 @@ # seconds) (integer value) #drydock_client_read_timeout = 300 +# Connect timeout used for connecting to Promenade using the Promenade client (in +# seconds) (integer value) +#promenade_client_connect_timeout = 20 + +# Read timeout used for responses from Promenade using the Promenade client (in +# seconds) (integer value) +#promenade_client_read_timeout = 300 + [shipyard] diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_validate_design.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_validate_design.py index 77f4a56e..59505526 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_validate_design.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_validate_design.py @@ -18,13 +18,17 @@ import requests from airflow.plugins_manager import AirflowPlugin from airflow.exceptions import AirflowException +from oslo_config import cfg +from shipyard_airflow.shipyard_const import CustomHeaders + +CONF = cfg.CONF try: from drydock_base_operator import DrydockBaseOperator except ImportError: from shipyard_airflow.plugins.drydock_base_operator import \ DrydockBaseOperator -from shipyard_airflow.shipyard_const import CustomHeaders + LOG = logging.getLogger(__name__) @@ -64,10 +68,13 @@ class DrydockValidateDesignOperator(DrydockBaseOperator): LOG.info("Waiting for DryDock to validate site design...") try: - design_validate_response = requests.post(validation_endpoint, - headers=headers, - data=json.dumps(payload), - timeout=5) + design_validate_response = requests.post( + validation_endpoint, + headers=headers, + data=json.dumps(payload), + timeout=( + CONF.requests_config.drydock_client_connect_timeout, + CONF.requests_config.drydock_client_read_timeout)) except requests.exceptions.RequestException as e: raise AirflowException(e) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py index 213b3573..1daf03f5 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py @@ -18,13 +18,17 @@ import requests from airflow.exceptions import AirflowException from airflow.plugins_manager import AirflowPlugin +from oslo_config import cfg +from shipyard_airflow.shipyard_const import CustomHeaders + +CONF = cfg.CONF try: from promenade_base_operator import PromenadeBaseOperator except ImportError: from shipyard_airflow.plugins.promenade_base_operator import \ PromenadeBaseOperator -from shipyard_airflow.shipyard_const import CustomHeaders + LOG = logging.getLogger(__name__) @@ -64,10 +68,13 @@ class PromenadeValidateSiteDesignOperator(PromenadeBaseOperator): LOG.info("Waiting for Promenade to validate site design...") try: - design_validate_response = requests.post(validation_endpoint, - headers=headers, - data=json.dumps(payload), - timeout=5) + design_validate_response = requests.post( + validation_endpoint, + headers=headers, + data=json.dumps(payload), + timeout=( + CONF.requests_config.drydock_client_connect_timeout, + CONF.requests_config.drydock_client_read_timeout)) except requests.exceptions.RequestException as e: raise AirflowException(e) diff --git a/src/bin/shipyard_client/shipyard_client/api_client/base_client.py b/src/bin/shipyard_client/shipyard_client/api_client/base_client.py index 5e64d871..c4308f70 100644 --- a/src/bin/shipyard_client/shipyard_client/api_client/base_client.py +++ b/src/bin/shipyard_client/shipyard_client/api_client/base_client.py @@ -19,6 +19,7 @@ from keystoneauth1.exceptions.auth import AuthorizationFailure from keystoneauth1.exceptions.catalog import EndpointNotFound from keystoneauth1.identity import v3 from keystoneauth1 import session +from oslo_config import cfg import requests from shipyard_client.api_client.client_error import ClientError @@ -27,6 +28,7 @@ from shipyard_client.api_client.client_error import UnauthorizedClientError from shipyard_client.api_client.client_error import ShipyardBufferError from shipyard_client.api_client.client_error import InvalidCollectionError +CONF = cfg.CONF class BaseClient(metaclass=abc.ABCMeta): """Abstract base client class @@ -95,7 +97,13 @@ class BaseClient(metaclass=abc.ABCMeta): # This could use keystoneauth1 session, but that library handles # responses strangely (wraps all 400/500 in a keystone exception) response = requests.post( - url, data=data, params=query_params, headers=headers) + url, + data=data, + params=query_params, + headers=headers, + timeout=( + CONF.requests_config.base_client_connect_timeout, + CONF.requests_config.base_client_read_timeout)) # handle some cases where the response code is sufficient to know # what needs to be done if response.status_code == 401: