airship/shipyard
Code Issues Proposed changes

Shipyard timeout issue

Browse Source 
This PS  adds default values for chart values and resolves some issues
in python code that utilizes these values:

      validation_connect_timeout: 20
      validation_read_timeout: 300
      deckhand_client_connect_timeout: 20
      deckhand_client_read_timeout: 300
      drydock_client_connect_timeout: 20
      drydock_client_read_timeout: 300

Change-Id: Ic5b1920257859239613a3ce77134e6b05bd7e9dd
This commit is contained in:
Sergiy Markin 2023-05-11 19:46:41 +00:00
parent 296853f159
commit 9c28c832dd
16 changed files with 102 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
charts/shipyard/values.yaml
View File

@ -415,12 +415,12 @@ conf:
requests_config:
airflow_log_connect_timeout: 5
airflow_log_read_timeout: 300
deckhand_client_connect_timeout: 5
deckhand_client_read_timeout: 300
validation_connect_timeout: 5
validation_connect_timeout: 20
validation_read_timeout: 300
notes_connect_timeout: 5
notes_read_timeout: 10
deckhand_client_connect_timeout: 20
deckhand_client_read_timeout: 300
drydock_client_connect_timeout: 20
drydock_client_read_timeout: 300
airflow:

14
doc/source/_static/shipyard.conf.sample
View File

@ -376,13 +376,6 @@
# Airflow logs retrieval timeout (in seconds) (integer value)
#airflow_log_read_timeout = 300
# Deckhand client connect timeout (in seconds) (integer value)
#deckhand_client_connect_timeout = 5
# Deckhand client timeout (in seconds) for GET, PUT, POST and DELETE request
# (integer value)
#deckhand_client_read_timeout = 300
# Airship component validation connect timeout (in seconds) (integer value)
#validation_connect_timeout = 5
@ -396,6 +389,13 @@
# Read timeout for a note source URL (in seconds) (integer value)
#notes_read_timeout = 10
# Deckhand client connect timeout (in seconds) (integer value)
#deckhand_client_connect_timeout = 5
# Deckhand client timeout (in seconds) for GET, PUT, POST and DELETE request
# (integer value)
#deckhand_client_read_timeout = 300
# Connect timeout used for connecting to Drydock using the Drydock client (in
# seconds) (integer value)
#drydock_client_connect_timeout = 20

14
src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample
View File

@ -376,13 +376,6 @@
# Airflow logs retrieval timeout (in seconds) (integer value)
#airflow_log_read_timeout = 300
# Deckhand client connect timeout (in seconds) (integer value)
#deckhand_client_connect_timeout = 5
# Deckhand client timeout (in seconds) for GET, PUT, POST and DELETE request
# (integer value)
#deckhand_client_read_timeout = 300
# Airship component validation connect timeout (in seconds) (integer value)
#validation_connect_timeout = 5
@ -396,6 +389,13 @@
# Read timeout for a note source URL (in seconds) (integer value)
#notes_read_timeout = 10
# Deckhand client connect timeout (in seconds) (integer value)
#deckhand_client_connect_timeout = 5
# Deckhand client timeout (in seconds) for GET, PUT, POST and DELETE request
# (integer value)
#deckhand_client_read_timeout = 300
# Connect timeout used for connecting to Drydock using the Drydock client (in
# seconds) (integer value)
#drydock_client_connect_timeout = 20

6
src/bin/shipyard_airflow/requirements-direct.txt
View File

@ -34,7 +34,7 @@ WTForms<=2.3.3
itsdangerous==2.0.1
git+https://opendev.org/airship/deckhand.git@e4cf0c17df103683b971b07f6b29d58f1ae681c0#egg=deckhand
git+https://opendev.org/airship/deckhand.git@3a06b1b6040865483a09514e2bf7dc5fae24560e#egg=deckhand
git+https://opendev.org/airship/drydock.git@375abedb8aa3a413ca70a47aef467f39d65e1aee#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@b778f9faf890dce74c3e5650fb3a196b5d25628b#egg=armada
git+https://opendev.org/airship/promenade.git@2e8f6642f446e4ea6af2d6854b0f87564ce0fe60#egg=promenade
git+https://opendev.org/airship/armada.git@4be7fa44a743fbcdf792e2bdca3af6c95b540f97#egg=armada
git+https://opendev.org/airship/promenade.git@45bcba068eb42fe6ba54d494b12122600dcb2c6c#egg=promenade

14
src/bin/shipyard_airflow/requirements-frozen.txt
View File

@ -4,7 +4,7 @@ amqp==2.6.1
apache-airflow==1.10.5
apispec==1.3.3
argcomplete==3.0.8
Armada @ git+https://opendev.org/airship/armada.git@b778f9faf890dce74c3e5650fb3a196b5d25628b
Armada @ git+https://opendev.org/airship/armada.git@4be7fa44a743fbcdf792e2bdca3af6c95b540f97
arrow==0.17.0
attrs==23.1.0
autopage==0.5.1
@ -29,7 +29,7 @@ coverage==5.3
croniter==0.3.37
cryptography==3.4.8
debtcollector==2.5.0
Deckhand @ git+https://opendev.org/airship/deckhand.git@e4cf0c17df103683b971b07f6b29d58f1ae681c0
Deckhand @ git+https://opendev.org/airship/deckhand.git@3a06b1b6040865483a09514e2bf7dc5fae24560e
decorator==5.1.1
deepdiff==5.8.1
defusedxml==0.7.1
@ -65,7 +65,7 @@ future==0.16.0
futurist==2.4.1
gitdb==4.0.10
GitPython==3.1.31
google-auth==2.17.3
google-auth==2.18.0
greenlet==2.0.2
gunicorn==19.10.0
hacking==4.1.0
@ -132,12 +132,12 @@ PasteDeploy==3.0.1
PasteScript==3.3.0
pbr==5.5.1
pendulum==1.4.4
pip==23.0.1
platformdirs==3.5.0
pip==23.1.2
platformdirs==3.5.1
ply==3.11
prettytable==3.7.0
prison==0.1.0
promenade @ git+https://opendev.org/airship/promenade.git@2e8f6642f446e4ea6af2d6854b0f87564ce0fe60
promenade @ git+https://opendev.org/airship/promenade.git@45bcba068eb42fe6ba54d494b12122600dcb2c6c
prometheus-client==0.8.0
protobuf==4.23.0
psutil==5.9.5
@ -202,7 +202,7 @@ SQLAlchemy==1.3.15
sqlalchemy-migrate==0.13.0
sqlparse==0.4.4
statsd==4.0.1
stevedore==5.0.0
stevedore==5.1.0
tabulate==0.8.3
Tempita==0.5.2
tenacity==4.12.0

26
src/bin/shipyard_airflow/shipyard_airflow/conf/config.py
View File

@ -210,19 +210,6 @@ SECTIONS = [
default=300,
help='Airflow logs retrieval timeout (in seconds)'
),
cfg.IntOpt(
'deckhand_client_connect_timeout',
default=5,
help='Deckhand client connect timeout (in seconds)'
),
cfg.IntOpt(
'deckhand_client_read_timeout',
default=300,
help=(
'Deckhand client timeout (in seconds) for GET, '
'PUT, POST and DELETE request'
)
),
cfg.IntOpt(
'validation_connect_timeout',
default=5,
@ -245,6 +232,19 @@ SECTIONS = [
default=10,
help='Read timeout for a note source URL (in seconds)'
),
cfg.IntOpt(
'deckhand_client_connect_timeout',
default=5,
help='Deckhand client connect timeout (in seconds)'
),
cfg.IntOpt(
'deckhand_client_read_timeout',
default=300,
help=(
'Deckhand client timeout (in seconds) for GET, '
'PUT, POST and DELETE request'
)
),
cfg.IntOpt(
'drydock_client_connect_timeout',
default=20,

7
src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_base_operator.py
View File

@ -72,10 +72,12 @@ class DeckhandBaseOperator(UcpBaseOperator):
*args, **kwargs)
self.committed_ver = committed_ver
self.deckhandclient = deckhandclient
self.deckhand_client_connect_timeout = None
self.deckhand_client_read_timeout = deckhand_client_read_timeout
self.revision_id = revision_id
self.svc_session = svc_session
self.svc_token = svc_token
self.validation_connect_timeout = None
self.validation_read_timeout = validation_read_timeout
@shipyard_service_token
@ -86,9 +88,12 @@ class DeckhandBaseOperator(UcpBaseOperator):
config.read(self.shipyard_conf)
# Initialize variables
self.deckhand_client_connect_timeout = int(config.get(
'requests_config', 'deckhand_client_connect_timeout'))
self.deckhand_client_read_timeout = int(config.get(
'requests_config', 'deckhand_client_read_timeout'))
self.validation_connect_timeout = int(config.get(
'requests_config', 'validation_connect_timeout'))
self.validation_read_timeout = int(config.get(
'requests_config', 'validation_read_timeout'))

3
src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_validate_site.py
View File

@ -54,7 +54,8 @@ class DeckhandValidateSiteDesignOperator(DeckhandBaseOperator):
retrieved_list = yaml.safe_load(
requests.get(validation_endpoint,
headers=x_auth_token,
timeout=self.validation_read_timeout).text)
timeout=(self.validation_connect_timeout,
self.validation_read_timeout)).text)
except requests.exceptions.RequestException as e:
raise AirflowException(e)

6
src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_base_operator.py
View File

@ -97,6 +97,8 @@ class DrydockBaseOperator(UcpBaseOperator):
self.svc_session = svc_session
self.svc_token = svc_token
self.target_nodes = None
self.validation_connect_timeout = None
self.validation_read_timeout = None
def run_base(self, context):
"""Base setup/processing for Drydock operators
@ -113,6 +115,10 @@ class DrydockBaseOperator(UcpBaseOperator):
'requests_config', 'drydock_client_connect_timeout'))
self.drydock_client_read_timeout = int(config.get(
'requests_config', 'drydock_client_read_timeout'))
self.validation_connect_timeout = int(config.get(
'requests_config', 'validation_connect_timeout'))
self.validation_read_timeout = int(config.get(
'requests_config', 'validation_read_timeout'))
# Setup the drydock client
self._setup_drydock_client()

13
src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_validate_design.py
View File

@ -18,13 +18,14 @@ import requests
from airflow.plugins_manager import AirflowPlugin
from airflow.exceptions import AirflowException
from shipyard_airflow.shipyard_const import CustomHeaders
try:
from drydock_base_operator import DrydockBaseOperator
except ImportError:
from shipyard_airflow.plugins.drydock_base_operator import \
DrydockBaseOperator
from shipyard_airflow.shipyard_const import CustomHeaders
LOG = logging.getLogger(__name__)
@ -64,10 +65,12 @@ class DrydockValidateDesignOperator(DrydockBaseOperator):
LOG.info("Waiting for DryDock to validate site design...")
try:
design_validate_response = requests.post(validation_endpoint,
headers=headers,
data=json.dumps(payload),
timeout=5)
design_validate_response = requests.post(
validation_endpoint,
headers=headers,
data=json.dumps(payload),
timeout=(self.validation_connect_timeout,
self.validation_read_timeout))
except requests.exceptions.RequestException as e:
raise AirflowException(e)

11
src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_base_operator.py
View File

@ -11,6 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import configparser
import logging
from airflow.plugins_manager import AirflowPlugin
@ -58,6 +59,8 @@ class PromenadeBaseOperator(UcpBaseOperator):
*args, **kwargs)
self.redeploy_server = redeploy_server
self.svc_token = svc_token
self.validation_connect_timeout = None
self.validation_read_timeout = None
@shipyard_service_token
def run_base(self, context):
@ -65,6 +68,14 @@ class PromenadeBaseOperator(UcpBaseOperator):
# Logs uuid of Shipyard action
LOG.info("Executing Shipyard Action %s", self.action_id)
# Retrieve config values from shipyard configuration.
config = configparser.ConfigParser()
config.read(self.shipyard_conf)
self.validation_connect_timeout = int(config.get(
'requests_config', 'validation_connect_timeout'))
self.validation_read_timeout = int(config.get(
'requests_config', 'validation_read_timeout'))
# Create additional headers dict to pass context marker
# and end user
addl_headers = {

14
src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py
View File

@ -18,13 +18,14 @@ import requests
from airflow.exceptions import AirflowException
from airflow.plugins_manager import AirflowPlugin
from shipyard_airflow.shipyard_const import CustomHeaders
try:
from promenade_base_operator import PromenadeBaseOperator
except ImportError:
from shipyard_airflow.plugins.promenade_base_operator import \
PromenadeBaseOperator
from shipyard_airflow.shipyard_const import CustomHeaders
LOG = logging.getLogger(__name__)
@ -64,10 +65,13 @@ class PromenadeValidateSiteDesignOperator(PromenadeBaseOperator):
LOG.info("Waiting for Promenade to validate site design...")
try:
design_validate_response = requests.post(validation_endpoint,
headers=headers,
data=json.dumps(payload),
timeout=5)
design_validate_response = requests.post(
validation_endpoint,
headers=headers,
data=json.dumps(payload),
timeout=(
self.validation_connect_timeout,
self.validation_read_timeout))
except requests.exceptions.RequestException as e:
raise AirflowException(e)

8
src/bin/shipyard_airflow/tests/unit/control/test.conf
View File

@ -36,10 +36,12 @@ service_type = kubernetesprovisioner
[requests_config]
airflow_log_connect_timeout = 5
airflow_log_read_timeout = 300
deckhand_client_connect_timeout = 5
deckhand_client_read_timeout = 300
validation_connect_timeout = 5
validation_connect_timeout = 20
validation_read_timeout = 300
deckhand_client_connect_timeout = 20
deckhand_client_read_timeout = 300
drydock_client_connect_timeout = 20
drydock_client_read_timeout = 300
[shipyard]
service_type = shipyard
[logging]

4
src/bin/shipyard_airflow/tests/unit/plugins/test.conf
View File

@ -10,6 +10,10 @@ profiler = false
[requests_config]
notes_connect_timeout = 5
notes_read_timeout = 10
validation_connect_timeout = 20
validation_read_timeout = 300
deckhand_client_connect_timeout = 20
deckhand_client_read_timeout = 300
drydock_client_connect_timeout = 20
drydock_client_read_timeout = 300

8
src/bin/shipyard_client/shipyard_client/api_client/base_client.py
View File

@ -14,12 +14,12 @@
import abc
import os
import logging
import requests
from keystoneauth1.exceptions.auth import AuthorizationFailure
from keystoneauth1.exceptions.catalog import EndpointNotFound
from keystoneauth1.identity import v3
from keystoneauth1 import session
import requests
from shipyard_client.api_client.client_error import ClientError
from shipyard_client.api_client.client_error import UnauthenticatedClientError
@ -95,7 +95,11 @@ class BaseClient(metaclass=abc.ABCMeta):
# This could use keystoneauth1 session, but that library handles
# responses strangely (wraps all 400/500 in a keystone exception)
response = requests.post(
url, data=data, params=query_params, headers=headers)
url,
data=data,
params=query_params,
headers=headers,
timeout=None)
# handle some cases where the response code is sufficient to know
# what needs to be done
if response.status_code == 401:

8
tools/resources/shipyard.conf
View File

@ -36,10 +36,12 @@ service_type = kubernetesprovisioner
[requests_config]
airflow_log_connect_timeout = 5
airflow_log_read_timeout = 300
deckhand_client_connect_timeout = 5
deckhand_client_read_timeout = 300
validation_connect_timeout = 5
validation_connect_timeout = 20
validation_read_timeout = 300
deckhand_client_connect_timeout = 20
deckhand_client_read_timeout = 300
drydock_client_connect_timeout = 20
drydock_client_read_timeout = 300
[shipyard]
service_type = shipyard
[oslo_policy]