From 7cc2e402f95371fa8d94295016127eb3b716508f Mon Sep 17 00:00:00 2001 From: Eric Harney Date: Wed, 6 Nov 2019 09:22:27 -0500 Subject: [PATCH] Fix remotefs clone volume locking Any method in the remotefs/nfs code that manipulates the qcow2 snapshot chain must be run separately from other methods that may touch this snapshot chain. This code intended to do this with a lock on the volume id, but it unintentionally locked only on the destination volume id rather than the source volume id where the snapshots are. This causes concurrent clone operations to fail in the NFS driver. This patch fixes this in the RemoteFSSnapDriverDistributed class which fixes the NFS driver and a handful of others. A follow up patch should be applied for the RemoteFSSnapDriver class with a similar change, but as that class is only used by one driver (which I can't test), this patch only adds a TODO for that change. Related-Bug: #1840712 Related-Bug: #1852449 Closes-Bug: #1851512 Change-Id: I64e041feaeb50c95808da46a34f334a9985018a8 --- cinder/volume/drivers/remotefs.py | 3 +++ ...motefs-clone-volume-locking-385e740d4a5a813b.yaml | 12 ++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 releasenotes/notes/fix-remotefs-clone-volume-locking-385e740d4a5a813b.yaml diff --git a/cinder/volume/drivers/remotefs.py b/cinder/volume/drivers/remotefs.py index 0cbece22249..36c1ce081de 100644 --- a/cinder/volume/drivers/remotefs.py +++ b/cinder/volume/drivers/remotefs.py @@ -1654,6 +1654,7 @@ class RemoteFSSnapDriver(RemoteFSSnapDriverBase): def create_volume_from_snapshot(self, volume, snapshot): return self._create_volume_from_snapshot(volume, snapshot) + # TODO: should be locking on src_vref id -- bug #1852449 @locked_volume_id_operation def create_cloned_volume(self, volume, src_vref): """Creates a clone of the specified volume.""" @@ -1696,6 +1697,8 @@ class RemoteFSSnapDriverDistributed(RemoteFSSnapDriverBase): def create_volume_from_snapshot(self, volume, snapshot): return self._create_volume_from_snapshot(volume, snapshot) + # lock the source volume id first + @coordination.synchronized('{self.driver_prefix}-{src_vref.id}') @coordination.synchronized('{self.driver_prefix}-{volume.id}') def create_cloned_volume(self, volume, src_vref): """Creates a clone of the specified volume.""" diff --git a/releasenotes/notes/fix-remotefs-clone-volume-locking-385e740d4a5a813b.yaml b/releasenotes/notes/fix-remotefs-clone-volume-locking-385e740d4a5a813b.yaml new file mode 100644 index 00000000000..dbf7b3dce18 --- /dev/null +++ b/releasenotes/notes/fix-remotefs-clone-volume-locking-385e740d4a5a813b.yaml @@ -0,0 +1,12 @@ +--- +fixes: + - | + An incorrect lock in the remotefs code, which is used for the NFS driver, + and other similar drivers, resulted in concurrent clone volume operations + failing. create_cloned_volume now locks on the source volume id, meaning + multiple clone operations from the same source volume are serialized. + + A lock in the volume manager flow generally prevents this on normal clone + volume operations, but this clone method in the driver is called for + operations such as cloning from the cinder image-volume cache or cloning + from a cinder backend used as a glance store.