From 411852892d803b606110d0956a59764925c16ec6 Mon Sep 17 00:00:00 2001 From: Gorka Eguileor Date: Wed, 6 Jul 2022 20:51:34 +0200 Subject: [PATCH] PowerMax: Fix deadlock moving SGs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There's a potential deadlock scenario in PowerMax's masking.py "do_move_volume_between_storage_groups" method. The method uses 2 locks, one for the source Storage Group and another for the destination Storage Group, and it could happen that if 2 requests going in opposite directions are received simultaneously their first lock acquisition interleaves resulting in a deadlock situation. @coordination.synchronized( "emc-sg-{source_storagegroup_name}-{serial_number}") @coordination.synchronized( "emc-sg-{target_storagegroup_name}-{serial_number}") def do_move_volume_between_storage_groups( serial_number, source_storage_group_name, target_storage_group_name): The scenario would be like this: - User requests an instance migration from A to B - User requests an instance migration from B to A - Driver acquires the first lock for A-to-B for example something like cinder-emc-sg-SGA-### - Driver acquires the first lock for B-to-A for example something like cinder-emc-sgSGB-### The deadlock happens because A-to-B waits forever for the lock held by the B-to-A operation, which in turn cannot proceed because it’s waiting for lock held by A-to-B. This patch fixes it using the new coordination.synchronized functionality that ensures that a series of locks are always acquired in the same order, preventing deadlocks. Closes-Bug: #1980870 Change-Id: I7eda4645575cfaedcf45d73ab3a215976d3fac3a --- cinder/volume/drivers/dell_emc/powermax/masking.py | 3 +-- releasenotes/notes/powermax-deadlock-5fdcacb63ca87159.yaml | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/powermax-deadlock-5fdcacb63ca87159.yaml diff --git a/cinder/volume/drivers/dell_emc/powermax/masking.py b/cinder/volume/drivers/dell_emc/powermax/masking.py index a23df8c4cd7..54b86b96f08 100644 --- a/cinder/volume/drivers/dell_emc/powermax/masking.py +++ b/cinder/volume/drivers/dell_emc/powermax/masking.py @@ -665,8 +665,7 @@ class PowerMaxMasking(object): return @coordination.synchronized( - "emc-sg-{source_storage_group_name}-{serial_number}") - @coordination.synchronized( + "emc-sg-{source_storage_group_name}-{serial_number}", "emc-sg-{target_storage_group_name}-{serial_number}") def do_move_volume_between_storage_groups( serial_number, source_storage_group_name, diff --git a/releasenotes/notes/powermax-deadlock-5fdcacb63ca87159.yaml b/releasenotes/notes/powermax-deadlock-5fdcacb63ca87159.yaml new file mode 100644 index 00000000000..718712b8676 --- /dev/null +++ b/releasenotes/notes/powermax-deadlock-5fdcacb63ca87159.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Dell EMC PowerMax driver `bug #1980870 + `_: Fixed potential + deadlock when moving volumes between Storage Groups.