From e05bb37ae2b91df0a8391fa1f3c6d91de24aa9f1 Mon Sep 17 00:00:00 2001 From: shenjiatong Date: Tue, 30 Jun 2020 08:56:26 +0800 Subject: [PATCH] Catch ConnectionForced Exception Occasionally I saw large number of connectionforced exceptions. such exceptions cannot be healed automatically and requires a reboot. catch it and ensure connection established. Changes: oslo_messaging/_drivers/impl_rabbit.py NOTE(stephenfin): We need to add the 'absolute_import' future to disable the legacy import flow. Without this, the import of 'amqp.exceptions' in 'oslo_messaging._drivers.impl_rabbit' attempts to import 'exceptions' from the sibling module, 'oslo_messaging._drivers.amqp', instead of the top-level 'amqp' module. Thank God for Python 3.x, eh? More information is available at [1], for those curious enough. NOTE(hberaud): conflicts within the cherry-pick on stable/stein due to removed log translation and i18n on stable/train [2]. I decided to ignore this patch [2] for stable/stein first to prepare cherry-picking on previous stable branches (rocky, queens) and secondly because I don't think we want it on older branches. [1] https://stackoverflow.com/q/33743880/ [2] https://opendev.org/openstack/oslo.messaging/commit/4f385720d7d43aadfaff7c6f41b0534312b23ba8 Change-Id: I5c468d10230f05438df780e045f1e92662edf070 Closes-Bug: #1883038 (cherry picked from commit 26e8bb4cc9097fddf3c2e34599b131e3f5d73867) (cherry picked from commit 360deb89a7e95d332267564b486219e389a793f7) (cherry picked from commit 54ced5361b49fb9b0cf53ff2e6f5825ba7ae513a) --- oslo_messaging/_drivers/impl_rabbit.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/oslo_messaging/_drivers/impl_rabbit.py b/oslo_messaging/_drivers/impl_rabbit.py index 54e4dc03e..122b16607 100644 --- a/oslo_messaging/_drivers/impl_rabbit.py +++ b/oslo_messaging/_drivers/impl_rabbit.py @@ -12,6 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. +from __future__ import absolute_import + import contextlib import errno import functools @@ -26,6 +28,7 @@ import threading import time import uuid +from amqp import exceptions as amqp_exec import kombu import kombu.connection import kombu.entity @@ -953,7 +956,8 @@ class Connection(object): except (socket.timeout, ConnectRefuseError, OSError, - kombu.exceptions.OperationalError) as exc: + kombu.exceptions.OperationalError, + amqp_exec.ConnectionForced) as exc: LOG.info(_LI("A recoverable connection/channel error " "occurred, trying to reconnect: %s"), exc) self.ensure_connection()