Shipyard timeout issue

This PS  adds default values for chart values and resolves some issues
in python code that utilizes these values:

      validation_connect_timeout: 20
      validation_read_timeout: 300
      deckhand_client_connect_timeout: 20
      deckhand_client_read_timeout: 300
      drydock_client_connect_timeout: 20
      drydock_client_read_timeout: 300

Change-Id: Ic5b1920257859239613a3ce77134e6b05bd7e9dd

charts/shipyard/values.yaml

@@ -415,12 +415,12 @@ conf:
     requests_config:
       airflow_log_connect_timeout: 5
       airflow_log_read_timeout: 300
-      deckhand_client_connect_timeout: 5
+      validation_connect_timeout: 20
-      deckhand_client_read_timeout: 300
-      validation_connect_timeout: 5
       validation_read_timeout: 300
       notes_connect_timeout: 5
       notes_read_timeout: 10
+      deckhand_client_connect_timeout: 20
+      deckhand_client_read_timeout: 300
       drydock_client_connect_timeout: 20
       drydock_client_read_timeout: 300
     airflow:

doc/source/_static/shipyard.conf.sample

@@ -376,13 +376,6 @@
 # Airflow logs retrieval timeout (in seconds) (integer value)
 #airflow_log_read_timeout = 300
 
-# Deckhand client connect timeout (in seconds) (integer value)
-#deckhand_client_connect_timeout = 5
-
-# Deckhand client timeout (in seconds) for GET, PUT, POST and DELETE request
-# (integer value)
-#deckhand_client_read_timeout = 300
-
 # Airship component validation connect timeout (in seconds) (integer value)
 #validation_connect_timeout = 5
 
@@ -396,6 +389,13 @@
 # Read timeout for a note source URL (in seconds) (integer value)
 #notes_read_timeout = 10
 
+# Deckhand client connect timeout (in seconds) (integer value)
+#deckhand_client_connect_timeout = 5
+
+# Deckhand client timeout (in seconds) for GET, PUT, POST and DELETE request
+# (integer value)
+#deckhand_client_read_timeout = 300
+
 # Connect timeout used for connecting to Drydock using the Drydock client (in
 # seconds) (integer value)
 #drydock_client_connect_timeout = 20

src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample

@@ -376,13 +376,6 @@
 # Airflow logs retrieval timeout (in seconds) (integer value)
 #airflow_log_read_timeout = 300
 
-# Deckhand client connect timeout (in seconds) (integer value)
-#deckhand_client_connect_timeout = 5
-
-# Deckhand client timeout (in seconds) for GET, PUT, POST and DELETE request
-# (integer value)
-#deckhand_client_read_timeout = 300
-
 # Airship component validation connect timeout (in seconds) (integer value)
 #validation_connect_timeout = 5
 
@@ -396,6 +389,13 @@
 # Read timeout for a note source URL (in seconds) (integer value)
 #notes_read_timeout = 10
 
+# Deckhand client connect timeout (in seconds) (integer value)
+#deckhand_client_connect_timeout = 5
+
+# Deckhand client timeout (in seconds) for GET, PUT, POST and DELETE request
+# (integer value)
+#deckhand_client_read_timeout = 300
+
 # Connect timeout used for connecting to Drydock using the Drydock client (in
 # seconds) (integer value)
 #drydock_client_connect_timeout = 20

src/bin/shipyard_airflow/requirements-direct.txt

@@ -34,7 +34,7 @@ WTForms<=2.3.3
 
 itsdangerous==2.0.1
 
-git+https://opendev.org/airship/deckhand.git@e4cf0c17df103683b971b07f6b29d58f1ae681c0#egg=deckhand
+git+https://opendev.org/airship/deckhand.git@3a06b1b6040865483a09514e2bf7dc5fae24560e#egg=deckhand
 git+https://opendev.org/airship/drydock.git@375abedb8aa3a413ca70a47aef467f39d65e1aee#egg=drydock_provisioner&subdirectory=python
-git+https://opendev.org/airship/armada.git@b778f9faf890dce74c3e5650fb3a196b5d25628b#egg=armada
+git+https://opendev.org/airship/armada.git@4be7fa44a743fbcdf792e2bdca3af6c95b540f97#egg=armada
-git+https://opendev.org/airship/promenade.git@2e8f6642f446e4ea6af2d6854b0f87564ce0fe60#egg=promenade
+git+https://opendev.org/airship/promenade.git@45bcba068eb42fe6ba54d494b12122600dcb2c6c#egg=promenade

src/bin/shipyard_airflow/requirements-frozen.txt

@@ -4,7 +4,7 @@ amqp==2.6.1
 apache-airflow==1.10.5
 apispec==1.3.3
 argcomplete==3.0.8
-Armada @ git+https://opendev.org/airship/armada.git@b778f9faf890dce74c3e5650fb3a196b5d25628b
+Armada @ git+https://opendev.org/airship/armada.git@4be7fa44a743fbcdf792e2bdca3af6c95b540f97
 arrow==0.17.0
 attrs==23.1.0
 autopage==0.5.1
@@ -29,7 +29,7 @@ coverage==5.3
 croniter==0.3.37
 cryptography==3.4.8
 debtcollector==2.5.0
-Deckhand @ git+https://opendev.org/airship/deckhand.git@e4cf0c17df103683b971b07f6b29d58f1ae681c0
+Deckhand @ git+https://opendev.org/airship/deckhand.git@3a06b1b6040865483a09514e2bf7dc5fae24560e
 decorator==5.1.1
 deepdiff==5.8.1
 defusedxml==0.7.1
@@ -65,7 +65,7 @@ future==0.16.0
 futurist==2.4.1
 gitdb==4.0.10
 GitPython==3.1.31
-google-auth==2.17.3
+google-auth==2.18.0
 greenlet==2.0.2
 gunicorn==19.10.0
 hacking==4.1.0
@@ -132,12 +132,12 @@ PasteDeploy==3.0.1
 PasteScript==3.3.0
 pbr==5.5.1
 pendulum==1.4.4
-pip==23.0.1
+pip==23.1.2
-platformdirs==3.5.0
+platformdirs==3.5.1
 ply==3.11
 prettytable==3.7.0
 prison==0.1.0
-promenade @ git+https://opendev.org/airship/promenade.git@2e8f6642f446e4ea6af2d6854b0f87564ce0fe60
+promenade @ git+https://opendev.org/airship/promenade.git@45bcba068eb42fe6ba54d494b12122600dcb2c6c
 prometheus-client==0.8.0
 protobuf==4.23.0
 psutil==5.9.5
@@ -202,7 +202,7 @@ SQLAlchemy==1.3.15
 sqlalchemy-migrate==0.13.0
 sqlparse==0.4.4
 statsd==4.0.1
-stevedore==5.0.0
+stevedore==5.1.0
 tabulate==0.8.3
 Tempita==0.5.2
 tenacity==4.12.0

src/bin/shipyard_airflow/shipyard_airflow/conf/config.py

@@ -210,19 +210,6 @@ SECTIONS = [
                 default=300,
                 help='Airflow logs retrieval timeout (in seconds)'
             ),
-            cfg.IntOpt(
-                'deckhand_client_connect_timeout',
-                default=5,
-                help='Deckhand client connect timeout (in seconds)'
-            ),
-            cfg.IntOpt(
-                'deckhand_client_read_timeout',
-                default=300,
-                help=(
-                    'Deckhand client timeout (in seconds) for GET, '
-                    'PUT, POST and DELETE request'
-                )
-            ),
             cfg.IntOpt(
                 'validation_connect_timeout',
                 default=5,
@@ -245,6 +232,19 @@ SECTIONS = [
                 default=10,
                 help='Read timeout for a note source URL (in seconds)'
             ),
+            cfg.IntOpt(
+                'deckhand_client_connect_timeout',
+                default=5,
+                help='Deckhand client connect timeout (in seconds)'
+            ),
+            cfg.IntOpt(
+                'deckhand_client_read_timeout',
+                default=300,
+                help=(
+                    'Deckhand client timeout (in seconds) for GET, '
+                    'PUT, POST and DELETE request'
+                )
+            ),
             cfg.IntOpt(
                 'drydock_client_connect_timeout',
                 default=20,

src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_base_operator.py

@@ -72,10 +72,12 @@ class DeckhandBaseOperator(UcpBaseOperator):
                   *args, **kwargs)
         self.committed_ver = committed_ver
         self.deckhandclient = deckhandclient
+        self.deckhand_client_connect_timeout = None
         self.deckhand_client_read_timeout = deckhand_client_read_timeout
         self.revision_id = revision_id
         self.svc_session = svc_session
         self.svc_token = svc_token
+        self.validation_connect_timeout = None
         self.validation_read_timeout = validation_read_timeout
 
     @shipyard_service_token
@@ -86,9 +88,12 @@ class DeckhandBaseOperator(UcpBaseOperator):
         config.read(self.shipyard_conf)
 
         # Initialize variables
+        self.deckhand_client_connect_timeout = int(config.get(
+            'requests_config', 'deckhand_client_connect_timeout'))
         self.deckhand_client_read_timeout = int(config.get(
             'requests_config', 'deckhand_client_read_timeout'))
-
+        self.validation_connect_timeout = int(config.get(
+            'requests_config', 'validation_connect_timeout'))
         self.validation_read_timeout = int(config.get(
             'requests_config', 'validation_read_timeout'))
 

src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_validate_site.py

@@ -54,7 +54,8 @@ class DeckhandValidateSiteDesignOperator(DeckhandBaseOperator):
             retrieved_list = yaml.safe_load(
                 requests.get(validation_endpoint,
                              headers=x_auth_token,
-                             timeout=self.validation_read_timeout).text)
+                             timeout=(self.validation_connect_timeout,
+                                      self.validation_read_timeout)).text)
 
         except requests.exceptions.RequestException as e:
             raise AirflowException(e)

src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_base_operator.py

@@ -97,6 +97,8 @@ class DrydockBaseOperator(UcpBaseOperator):
         self.svc_session = svc_session
         self.svc_token = svc_token
         self.target_nodes = None
+        self.validation_connect_timeout = None
+        self.validation_read_timeout = None
 
     def run_base(self, context):
         """Base setup/processing for Drydock operators
@@ -113,6 +115,10 @@ class DrydockBaseOperator(UcpBaseOperator):
                 'requests_config', 'drydock_client_connect_timeout'))
             self.drydock_client_read_timeout = int(config.get(
                 'requests_config', 'drydock_client_read_timeout'))
+            self.validation_connect_timeout = int(config.get(
+                'requests_config', 'validation_connect_timeout'))
+            self.validation_read_timeout = int(config.get(
+                'requests_config', 'validation_read_timeout'))
 
             # Setup the drydock client
             self._setup_drydock_client()

src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_validate_design.py

@@ -18,13 +18,14 @@ import requests
 
 from airflow.plugins_manager import AirflowPlugin
 from airflow.exceptions import AirflowException
+from shipyard_airflow.shipyard_const import CustomHeaders
 
 try:
     from drydock_base_operator import DrydockBaseOperator
 except ImportError:
     from shipyard_airflow.plugins.drydock_base_operator import \
         DrydockBaseOperator
-from shipyard_airflow.shipyard_const import CustomHeaders
+
 
 LOG = logging.getLogger(__name__)
 
@@ -64,10 +65,12 @@ class DrydockValidateDesignOperator(DrydockBaseOperator):
         LOG.info("Waiting for DryDock to validate site design...")
 
         try:
-            design_validate_response = requests.post(validation_endpoint,
+            design_validate_response = requests.post(
-                                                     headers=headers,
+                validation_endpoint,
-                                                     data=json.dumps(payload),
+                headers=headers,
-                                                     timeout=5)
+                data=json.dumps(payload),
+                timeout=(self.validation_connect_timeout,
+                         self.validation_read_timeout))
 
         except requests.exceptions.RequestException as e:
             raise AirflowException(e)

src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_base_operator.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import configparser
 import logging
 
 from airflow.plugins_manager import AirflowPlugin
@@ -58,6 +59,8 @@ class PromenadeBaseOperator(UcpBaseOperator):
                   *args, **kwargs)
         self.redeploy_server = redeploy_server
         self.svc_token = svc_token
+        self.validation_connect_timeout = None
+        self.validation_read_timeout = None
 
     @shipyard_service_token
     def run_base(self, context):
@@ -65,6 +68,14 @@ class PromenadeBaseOperator(UcpBaseOperator):
         # Logs uuid of Shipyard action
         LOG.info("Executing Shipyard Action %s", self.action_id)
 
+        # Retrieve config values from shipyard configuration.
+        config = configparser.ConfigParser()
+        config.read(self.shipyard_conf)
+        self.validation_connect_timeout = int(config.get(
+            'requests_config', 'validation_connect_timeout'))
+        self.validation_read_timeout = int(config.get(
+            'requests_config', 'validation_read_timeout'))
+
         # Create additional headers dict to pass context marker
         # and end user
         addl_headers = {

src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py

@@ -18,13 +18,14 @@ import requests
 
 from airflow.exceptions import AirflowException
 from airflow.plugins_manager import AirflowPlugin
+from shipyard_airflow.shipyard_const import CustomHeaders
 
 try:
     from promenade_base_operator import PromenadeBaseOperator
 except ImportError:
     from shipyard_airflow.plugins.promenade_base_operator import \
         PromenadeBaseOperator
-from shipyard_airflow.shipyard_const import CustomHeaders
+
 
 LOG = logging.getLogger(__name__)
 
@@ -64,10 +65,13 @@ class PromenadeValidateSiteDesignOperator(PromenadeBaseOperator):
         LOG.info("Waiting for Promenade to validate site design...")
 
         try:
-            design_validate_response = requests.post(validation_endpoint,
+            design_validate_response = requests.post(
-                                                     headers=headers,
+                validation_endpoint,
-                                                     data=json.dumps(payload),
+                headers=headers,
-                                                     timeout=5)
+                data=json.dumps(payload),
+                timeout=(
+                    self.validation_connect_timeout,
+                    self.validation_read_timeout))
 
         except requests.exceptions.RequestException as e:
             raise AirflowException(e)

src/bin/shipyard_airflow/tests/unit/control/test.conf

@@ -36,10 +36,12 @@ service_type = kubernetesprovisioner
 [requests_config]
 airflow_log_connect_timeout = 5
 airflow_log_read_timeout = 300
-deckhand_client_connect_timeout = 5
+validation_connect_timeout = 20
-deckhand_client_read_timeout = 300
-validation_connect_timeout = 5
 validation_read_timeout = 300
+deckhand_client_connect_timeout = 20
+deckhand_client_read_timeout = 300
+drydock_client_connect_timeout = 20
+drydock_client_read_timeout = 300
 [shipyard]
 service_type = shipyard
 [logging]

src/bin/shipyard_airflow/tests/unit/plugins/test.conf

@@ -10,6 +10,10 @@ profiler = false
 [requests_config]
 notes_connect_timeout = 5
 notes_read_timeout = 10
+validation_connect_timeout = 20
+validation_read_timeout = 300
+deckhand_client_connect_timeout = 20
+deckhand_client_read_timeout = 300
 drydock_client_connect_timeout = 20
 drydock_client_read_timeout = 300
 

src/bin/shipyard_client/shipyard_client/api_client/base_client.py

@@ -14,12 +14,12 @@
 import abc
 import os
 import logging
+import requests
 
 from keystoneauth1.exceptions.auth import AuthorizationFailure
 from keystoneauth1.exceptions.catalog import EndpointNotFound
 from keystoneauth1.identity import v3
 from keystoneauth1 import session
-import requests
 
 from shipyard_client.api_client.client_error import ClientError
 from shipyard_client.api_client.client_error import UnauthenticatedClientError
@@ -95,7 +95,11 @@ class BaseClient(metaclass=abc.ABCMeta):
             # This could use keystoneauth1 session, but that library handles
             # responses strangely (wraps all 400/500 in a keystone exception)
             response = requests.post(
-                url, data=data, params=query_params, headers=headers)
+                url,
+                data=data,
+                params=query_params,
+                headers=headers,
+                timeout=None)
             # handle some cases where the response code is sufficient to know
             # what needs to be done
             if response.status_code == 401:

tools/resources/shipyard.conf

@@ -36,10 +36,12 @@ service_type = kubernetesprovisioner
 [requests_config]
 airflow_log_connect_timeout = 5
 airflow_log_read_timeout = 300
-deckhand_client_connect_timeout = 5
+validation_connect_timeout = 20
-deckhand_client_read_timeout = 300
-validation_connect_timeout = 5
 validation_read_timeout = 300
+deckhand_client_connect_timeout = 20
+deckhand_client_read_timeout = 300
+drydock_client_connect_timeout = 20
+drydock_client_read_timeout = 300
 [shipyard]
 service_type = shipyard
 [oslo_policy]