From 4e75297b58c4a488451d575819608ec203c89972 Mon Sep 17 00:00:00 2001 From: Andy Ning Date: Mon, 10 May 2021 12:24:52 -0400 Subject: [PATCH] Specify default connect_timeout for sysinv agent During bootstrap, for some reason rabbitMQ takes longer time to response to sysinv agent's AMQP request after TCP connection is established. This usually happens after multiple connection failures, and can be as long as 8s. But it's still within th 10s handshake_timeout configured on rabbitMQ. However the sysinv agent uses the default 5s timeout from kombu lib so it doesn't wait long enough before hanging up the connection. This causes the agent can't reconnect to the rabbitMQ. In turn, the defered manifests don't have chance to pass readiness checking and get applied. This commit specifies a 10s default connect_timeout for the agent t0 connect to rabbitMQ, aligning with the server side timeout. Closes-Bug: 1928008 Signed-off-by: Andy Ning Change-Id: I8cc476910f47fca687ddcfd5f3d20f451f70ffce --- .../sysinv/sysinv/sysinv/openstack/common/rpc/impl_kombu.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sysinv/sysinv/sysinv/sysinv/openstack/common/rpc/impl_kombu.py b/sysinv/sysinv/sysinv/sysinv/openstack/common/rpc/impl_kombu.py index fcef36a617..31eee91e3a 100644 --- a/sysinv/sysinv/sysinv/sysinv/openstack/common/rpc/impl_kombu.py +++ b/sysinv/sysinv/sysinv/sysinv/openstack/common/rpc/impl_kombu.py @@ -82,6 +82,9 @@ kombu_opts = [ default=0, help='maximum retries with trying to connect to RabbitMQ ' '(the default of 0 implies an infinite retry count)'), + cfg.IntOpt('rabbit_connect_timeout', + default=10, + help='Timeout in seconds for connecting to the server'), cfg.BoolOpt('rabbit_durable_queues', default=False, help='use durable queues in RabbitMQ'), @@ -426,6 +429,7 @@ class Connection(object): 'userid': self.conf.rabbit_userid, 'password': self.conf.rabbit_password, 'virtual_host': self.conf.rabbit_virtual_host, + 'connect_timeout': self.conf.rabbit_connect_timeout, } for sp_key, value in server_params.items():