Shipyard API timeout issue

This PS  adds default values for chart values and adds python code that
utilizes these values:

      drydock_client_connect_timeout: 20
      drydock_client_read_timeout: 300
      promenade_client_connect_timeout: 20
      promenade_client_read_timeout: 300

Also adjusting timeout in shipyard_client requests usinf hardcoded values:

- connection timeout - 5 sec
- read timeout - 300 seconds

Change-Id: Ic5b1920257859239613a3ce77134e6b05bd7e9dd

charts/shipyard/values.yaml

@@ -423,6 +423,10 @@ conf:
       notes_read_timeout: 10
       drydock_client_connect_timeout: 20
       drydock_client_read_timeout: 300
+      promenade_client_connect_timeout: 20
+      promenade_client_read_timeout: 300
+      shipyard_client_connect_timeout: 5
+      shipyard_client_read_timeout: 300
     airflow:
       worker_endpoint_scheme: 'http'
       worker_port: 8793

src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample

@@ -404,6 +404,14 @@
 # seconds) (integer value)
 #drydock_client_read_timeout = 300
 
+# Connect timeout used for connecting to Promenade using the Promenade client (in
+# seconds) (integer value)
+#promenade_client_connect_timeout = 20
+
+# Read timeout used for responses from Promenade using the Promenade client (in
+# seconds) (integer value)
+#promenade_client_read_timeout = 300
+
 
 [shipyard]
 

src/bin/shipyard_airflow/shipyard_airflow/conf/config.py

@@ -257,6 +257,18 @@ SECTIONS = [
                 help=('Read timeout used for responses from Drydock using '
                       'the Drydock client (in seconds)')
             ),
+            cfg.IntOpt(
+                'promenade_client_connect_timeout',
+                default=20,
+                help=('Connect timeout used for connecting to Drydock using '
+                      'the Drydock client (in seconds)')
+            ),
+            cfg.IntOpt(
+                'promenade_client_read_timeout',
+                default=300,
+                help=('Read timeout used for responses from Drydock using '
+                      'the Promenade client (in seconds)')
+            ),
         ]
     ),
     ConfigSection(

src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_validate_design.py

@@ -18,13 +18,17 @@ import requests
 
 from airflow.plugins_manager import AirflowPlugin
 from airflow.exceptions import AirflowException
+from oslo_config import cfg
+from shipyard_airflow.shipyard_const import CustomHeaders
+
+CONF = cfg.CONF
 
 try:
     from drydock_base_operator import DrydockBaseOperator
 except ImportError:
     from shipyard_airflow.plugins.drydock_base_operator import \
         DrydockBaseOperator
-from shipyard_airflow.shipyard_const import CustomHeaders
+
 
 LOG = logging.getLogger(__name__)
 
@@ -64,10 +68,13 @@ class DrydockValidateDesignOperator(DrydockBaseOperator):
         LOG.info("Waiting for DryDock to validate site design...")
 
         try:
-            design_validate_response = requests.post(validation_endpoint,
-                                                     headers=headers,
-                                                     data=json.dumps(payload),
-                                                     timeout=5)
+            design_validate_response = requests.post(
+                validation_endpoint,
+                headers=headers,
+                data=json.dumps(payload),
+                timeout=(
+                    CONF.requests_config.drydock_client_connect_timeout,
+                    CONF.requests_config.drydock_client_read_timeout))
 
         except requests.exceptions.RequestException as e:
             raise AirflowException(e)

src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py

@@ -18,13 +18,17 @@ import requests
 
 from airflow.exceptions import AirflowException
 from airflow.plugins_manager import AirflowPlugin
+from oslo_config import cfg
+from shipyard_airflow.shipyard_const import CustomHeaders
+
+CONF = cfg.CONF
 
 try:
     from promenade_base_operator import PromenadeBaseOperator
 except ImportError:
     from shipyard_airflow.plugins.promenade_base_operator import \
         PromenadeBaseOperator
-from shipyard_airflow.shipyard_const import CustomHeaders
+
 
 LOG = logging.getLogger(__name__)
 
@@ -64,10 +68,13 @@ class PromenadeValidateSiteDesignOperator(PromenadeBaseOperator):
         LOG.info("Waiting for Promenade to validate site design...")
 
         try:
-            design_validate_response = requests.post(validation_endpoint,
-                                                     headers=headers,
-                                                     data=json.dumps(payload),
-                                                     timeout=5)
+            design_validate_response = requests.post(
+                validation_endpoint,
+                headers=headers,
+                data=json.dumps(payload),
+                timeout=(
+                    CONF.requests_config.drydock_client_connect_timeout,
+                    CONF.requests_config.drydock_client_read_timeout))
 
         except requests.exceptions.RequestException as e:
             raise AirflowException(e)

src/bin/shipyard_client/shipyard_client/api_client/base_client.py

@@ -14,12 +14,12 @@
 import abc
 import os
 import logging
+import requests
 
 from keystoneauth1.exceptions.auth import AuthorizationFailure
 from keystoneauth1.exceptions.catalog import EndpointNotFound
 from keystoneauth1.identity import v3
 from keystoneauth1 import session
-import requests
 
 from shipyard_client.api_client.client_error import ClientError
 from shipyard_client.api_client.client_error import UnauthenticatedClientError
@@ -95,7 +95,11 @@ class BaseClient(metaclass=abc.ABCMeta):
             # This could use keystoneauth1 session, but that library handles
             # responses strangely (wraps all 400/500 in a keystone exception)
             response = requests.post(
-                url, data=data, params=query_params, headers=headers)
+                url,
+                data=data,
+                params=query_params,
+                headers=headers,
+                timeout=(5,300))
             # handle some cases where the response code is sufficient to know
             # what needs to be done
             if response.status_code == 401: