Shipyard API timeout issue

This PS  adds code that uses preconfigured drydock-api request timeout and adds these values for promenade as well as using them in the code:

      drydock_client_connect_timeout: 20
      drydock_client_read_timeout: 300
      promenade_client_connect_timeout: 20
      promenade_client_read_timeout: 300

Change-Id: Ic5b1920257859239613a3ce77134e6b05bd7e9dd

charts/shipyard/values.yaml

@@ -413,6 +413,8 @@ conf:
       auth_version: v3
       memcache_security_strategy: ENCRYPT
     requests_config:
+      base_client_connect_timeout: 5
+      base_client_read_timeout: 300
       airflow_log_connect_timeout: 5
       airflow_log_read_timeout: 300
       deckhand_client_connect_timeout: 5
@@ -423,6 +425,8 @@ conf:
       notes_read_timeout: 10
       drydock_client_connect_timeout: 20
       drydock_client_read_timeout: 300
+      promenade_client_connect_timeout: 20
+      promenade_client_read_timeout: 300
     airflow:
       worker_endpoint_scheme: 'http'
       worker_port: 8793

src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample

@@ -404,6 +404,14 @@
 # seconds) (integer value)
 #drydock_client_read_timeout = 300
 
+# Connect timeout used for connecting to Promenade using the Promenade client (in
+# seconds) (integer value)
+#promenade_client_connect_timeout = 20
+
+# Read timeout used for responses from Promenade using the Promenade client (in
+# seconds) (integer value)
+#promenade_client_read_timeout = 300
+
 
 [shipyard]
 

src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_validate_design.py

@@ -18,13 +18,17 @@ import requests
 
 from airflow.plugins_manager import AirflowPlugin
 from airflow.exceptions import AirflowException
+from oslo_config import cfg
+from shipyard_airflow.shipyard_const import CustomHeaders
+
+CONF = cfg.CONF
 
 try:
     from drydock_base_operator import DrydockBaseOperator
 except ImportError:
     from shipyard_airflow.plugins.drydock_base_operator import \
         DrydockBaseOperator
-from shipyard_airflow.shipyard_const import CustomHeaders
+
 
 LOG = logging.getLogger(__name__)
 
@@ -64,10 +68,13 @@ class DrydockValidateDesignOperator(DrydockBaseOperator):
         LOG.info("Waiting for DryDock to validate site design...")
 
         try:
-            design_validate_response = requests.post(validation_endpoint,
-                                                     headers=headers,
-                                                     data=json.dumps(payload),
-                                                     timeout=5)
+            design_validate_response = requests.post(
+                validation_endpoint,
+                headers=headers,
+                data=json.dumps(payload),
+                timeout=(
+                    CONF.requests_config.drydock_client_connect_timeout,
+                    CONF.requests_config.drydock_client_read_timeout))
 
         except requests.exceptions.RequestException as e:
             raise AirflowException(e)

src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py

@@ -18,13 +18,17 @@ import requests
 
 from airflow.exceptions import AirflowException
 from airflow.plugins_manager import AirflowPlugin
+from oslo_config import cfg
+from shipyard_airflow.shipyard_const import CustomHeaders
+
+CONF = cfg.CONF
 
 try:
     from promenade_base_operator import PromenadeBaseOperator
 except ImportError:
     from shipyard_airflow.plugins.promenade_base_operator import \
         PromenadeBaseOperator
-from shipyard_airflow.shipyard_const import CustomHeaders
+
 
 LOG = logging.getLogger(__name__)
 
@@ -64,10 +68,13 @@ class PromenadeValidateSiteDesignOperator(PromenadeBaseOperator):
         LOG.info("Waiting for Promenade to validate site design...")
 
         try:
-            design_validate_response = requests.post(validation_endpoint,
-                                                     headers=headers,
-                                                     data=json.dumps(payload),
-                                                     timeout=5)
+            design_validate_response = requests.post(
+                validation_endpoint,
+                headers=headers,
+                data=json.dumps(payload),
+                timeout=(
+                    CONF.requests_config.drydock_client_connect_timeout,
+                    CONF.requests_config.drydock_client_read_timeout))
 
         except requests.exceptions.RequestException as e:
             raise AirflowException(e)

src/bin/shipyard_client/shipyard_client/api_client/base_client.py

@@ -19,6 +19,7 @@ from keystoneauth1.exceptions.auth import AuthorizationFailure
 from keystoneauth1.exceptions.catalog import EndpointNotFound
 from keystoneauth1.identity import v3
 from keystoneauth1 import session
+from oslo_config import cfg
 import requests
 
 from shipyard_client.api_client.client_error import ClientError
@@ -27,6 +28,7 @@ from shipyard_client.api_client.client_error import UnauthorizedClientError
 from shipyard_client.api_client.client_error import ShipyardBufferError
 from shipyard_client.api_client.client_error import InvalidCollectionError
 
+CONF = cfg.CONF
 
 class BaseClient(metaclass=abc.ABCMeta):
     """Abstract base client class
@@ -95,7 +97,13 @@ class BaseClient(metaclass=abc.ABCMeta):
             # This could use keystoneauth1 session, but that library handles
             # responses strangely (wraps all 400/500 in a keystone exception)
             response = requests.post(
-                url, data=data, params=query_params, headers=headers)
+                url,
+                data=data,
+                params=query_params,
+                headers=headers,
+                timeout=(
+                    CONF.requests_config.base_client_connect_timeout,
+                    CONF.requests_config.base_client_read_timeout))
             # handle some cases where the response code is sufficient to know
             # what needs to be done
             if response.status_code == 401: