[Pure Storage] Add replication support for NVMe driver

Update driver with replication support prior to initial
release of this driver in Zed.

Change-Id: I75be679ec3cabe0d93534ec3f0115875295db630

cinder/tests/unit/volume/drivers/test_pure.py

@@ -87,6 +87,10 @@ FC_PORT_NAMES = ["ct0.fc2", "ct0.fc3", "ct1.fc2", "ct1.fc3"]
 NVME_IPS = ["10.0.0." + str(i + 1) for i in range(len(NVME_PORT_NAMES))]
 NVME_IPS += ["[2001:db8::" + str(i + 1) + "]"
              for i in range(len(NVME_PORT_NAMES))]
+AC_NVME_IPS = ["10.0.0." + str(i + 1 + len(NVME_PORT_NAMES))
+               for i in range(len(NVME_PORT_NAMES))]
+AC_NVME_IPS += ["[2001:db8::1:" + str(i + 1) + "]"
+                for i in range(len(NVME_PORT_NAMES))]
 NVME_CIDR = "0.0.0.0/0"
 NVME_CIDR_V6 = "::/0"
 NVME_PORT = 4420
@@ -131,6 +135,7 @@ NVME_CONNECTOR = {"nqn": INITIATOR_NQN, "host": HOSTNAME}
 ISCSI_CONNECTOR = {"initiator": INITIATOR_IQN, "host": HOSTNAME}
 FC_CONNECTOR = {"wwpns": {INITIATOR_WWN}, "host": HOSTNAME}
 TARGET_NQN = "nqn.2010-06.com.purestorage:flasharray.12345abc"
+AC_TARGET_NQN = "nqn.2010-06.com.purestorage:flasharray.67890def"
 TARGET_IQN = "iqn.2010-06.com.purestorage:flasharray.12345abc"
 AC_TARGET_IQN = "iqn.2018-06.com.purestorage:flasharray.67890def"
 TARGET_WWN = "21000024ff59fe94"
@@ -166,6 +171,12 @@ NVME_PORTS = [{"name": name,
                "portal": ip + ":" + TARGET_ROCE_PORT,
                "wwn": None,
                } for name, ip in zip(NVME_PORT_NAMES * 2, NVME_IPS)]
+AC_NVME_PORTS = [{"name": name,
+                  "nqn": AC_TARGET_NQN,
+                  "iqn": None,
+                  "portal": ip + ":" + TARGET_ROCE_PORT,
+                  "wwn": None,
+                  } for name, ip in zip(NVME_PORT_NAMES * 2, AC_NVME_IPS)]
 ISCSI_PORTS = [{"name": name,
                 "iqn": TARGET_IQN,
                 "portal": ip + ":" + TARGET_PORT,
@@ -340,7 +351,55 @@ NVME_CONNECTION_INFO_V6 = {
         "volume_nguid": "0009714b5cb916324a9374c470002b2c8",
     },
 }
-
+NVME_CONNECTION_INFO_AC = {
+    "driver_volume_type": "nvmeof",
+    "data": {
+        "target_nqn": TARGET_NQN,
+        "discard": True,
+        "portals": [
+            (NVME_IPS[0], NVME_PORT, "rdma"),
+            (NVME_IPS[1], NVME_PORT, "rdma"),
+            (NVME_IPS[2], NVME_PORT, "rdma"),
+            (NVME_IPS[3], NVME_PORT, "rdma"),
+            (AC_NVME_IPS[0], NVME_PORT, "rdma"),
+            (AC_NVME_IPS[1], NVME_PORT, "rdma"),
+            (AC_NVME_IPS[2], NVME_PORT, "rdma"),
+            (AC_NVME_IPS[3], NVME_PORT, "rdma")],
+        "volume_nguid": "0009714b5cb916324a9374c470002b2c8",
+    },
+}
+NVME_CONNECTION_INFO_AC_FILTERED = {
+    "driver_volume_type": "nvmeof",
+    "data": {
+        "target_nqn": TARGET_NQN,
+        "discard": True,
+        # Final entry filtered by NVME_CIDR_FILTERED
+        "portals": [
+            (NVME_IPS[0], NVME_PORT, "rdma"),
+            (NVME_IPS[1], NVME_PORT, "rdma"),
+            (NVME_IPS[2], NVME_PORT, "rdma"),
+            (NVME_IPS[3], NVME_PORT, "rdma"),
+            (AC_NVME_IPS[0], NVME_PORT, "rdma"),
+            (AC_NVME_IPS[1], NVME_PORT, "rdma"),
+            (AC_NVME_IPS[2], NVME_PORT, "rdma")],
+        "volume_nguid": "0009714b5cb916324a9374c470002b2c8",
+    },
+}
+NVME_CONNECTION_INFO_AC_FILTERED_LIST = {
+    "driver_volume_type": "nvmeof",
+    "data": {
+        "target_nqn": TARGET_NQN,
+        "discard": True,
+        # Final entry filtered by NVME_CIDR_FILTERED
+        "portals": [
+            (NVME_IPS[1], NVME_PORT, "rdma"),
+            (NVME_IPS[2], NVME_PORT, "rdma"),
+            (AC_NVME_IPS[5].strip("[]"), NVME_PORT, "rdma"),  # IPv6
+            (AC_NVME_IPS[6].strip("[]"), NVME_PORT, "rdma"),  # IPv6
+        ],
+        "volume_nguid": "0009714b5cb916324a9374c470002b2c8",
+    },
+}
 FC_CONNECTION_INFO = {
     "driver_volume_type": "fibre_channel",
     "data": {
@@ -978,37 +1037,6 @@ class PureBaseVolumeDriverTestCase(PureBaseSharedDriverTestCase):
             mock.call(self.array, [mock_sync_target], 'cinder-pod')
         ])
 
-    @mock.patch(BASE_DRIVER_OBJ + '._setup_replicated_pods')
-    @mock.patch(BASE_DRIVER_OBJ + '._generate_replication_retention')
-    @mock.patch(BASE_DRIVER_OBJ + '._setup_replicated_pgroups')
-    def test_do_setup_replicated_sync_rep_bad_driver(
-            self,
-            mock_setup_repl_pgroups,
-            mock_generate_replication_retention,
-            mock_setup_pods):
-        retention = mock.MagicMock()
-        mock_generate_replication_retention.return_value = retention
-        self._setup_mocks_for_replication()
-
-        self.mock_config.safe_get.return_value = [
-            {
-                "backend_id": "foo",
-                "managed_backend_name": None,
-                "san_ip": "1.2.3.4",
-                "api_token": "abc123",
-                "type": "sync",
-            }
-        ]
-        mock_sync_target = mock.MagicMock()
-        mock_sync_target.get.return_value = GET_ARRAY_SECONDARY
-        self.array.get.return_value = GET_ARRAY_PRIMARY
-        self.driver._storage_protocol = 'NVMe-RoCE'
-        self.purestorage_module.FlashArray.side_effect = [self.array,
-                                                          mock_sync_target]
-        self.assertRaises(pure.PureDriverException,
-                          self.driver.do_setup,
-                          None)
-
     def test_update_provider_info_update_all(self):
         test_vols = [
             self.new_fake_vol(spec={'id': fake.VOLUME_ID},
@@ -4520,6 +4548,155 @@ class PureNVMEDriverTestCase(PureBaseSharedDriverTestCase):
             NVME_CONNECTOR,
         )
 
+    @mock.patch(NVME_DRIVER_OBJ + "._get_nguid")
+    @mock.patch(NVME_DRIVER_OBJ + "._get_wwn")
+    @mock.patch(NVME_DRIVER_OBJ + "._connect")
+    @mock.patch(NVME_DRIVER_OBJ + "._get_target_nvme_ports")
+    def test_initialize_connection_uniform_ac(
+        self, mock_get_nvme_ports, mock_connection, mock_get_wwn,
+        mock_get_nguid
+    ):
+        repl_extra_specs = {
+            "replication_type": "<in> sync",
+            "replication_enabled": "<is> true",
+        }
+        vol, vol_name = self.new_fake_vol(type_extra_specs=repl_extra_specs)
+        mock_get_nvme_ports.side_effect = [NVME_PORTS, AC_NVME_PORTS]
+        mock_get_wwn.return_value = "3624a93709714b5cb91634c470002b2c8"
+        mock_get_nguid.return_value = "0009714b5cb916324a9374c470002b2c8"
+        mock_connection.side_effect = [
+            {
+                "vol": vol_name,
+                "lun": 1,
+            },
+            {
+                "vol": vol_name,
+                "lun": 5,
+            },
+        ]
+        result = deepcopy(NVME_CONNECTION_INFO_AC)
+
+        self.driver._is_active_cluster_enabled = True
+        mock_secondary = mock.MagicMock()
+        self.driver._uniform_active_cluster_target_arrays = [mock_secondary]
+
+        real_result = self.driver.initialize_connection(vol, NVME_CONNECTOR)
+        self.assertDictEqual(result, real_result)
+        mock_get_nvme_ports.assert_has_calls(
+            [
+                mock.call(self.array),
+                mock.call(mock_secondary),
+            ]
+        )
+        mock_connection.assert_has_calls(
+            [
+                mock.call(self.array, vol_name, NVME_CONNECTOR),
+                mock.call(
+                    mock_secondary, vol_name, NVME_CONNECTOR),
+            ]
+        )
+
+    @mock.patch(NVME_DRIVER_OBJ + "._get_nguid")
+    @mock.patch(NVME_DRIVER_OBJ + "._get_wwn")
+    @mock.patch(NVME_DRIVER_OBJ + "._connect")
+    @mock.patch(NVME_DRIVER_OBJ + "._get_target_nvme_ports")
+    def test_initialize_connection_uniform_ac_cidr(
+        self, mock_get_nvme_ports, mock_connection, mock_get_wwn,
+        mock_get_nguid
+    ):
+        repl_extra_specs = {
+            "replication_type": "<in> sync",
+            "replication_enabled": "<is> true",
+        }
+        vol, vol_name = self.new_fake_vol(type_extra_specs=repl_extra_specs)
+        mock_get_nvme_ports.side_effect = [NVME_PORTS, AC_NVME_PORTS]
+        mock_get_wwn.return_value = "3624a93709714b5cb91634c470002b2c8"
+        mock_get_nguid.return_value = "0009714b5cb916324a9374c470002b2c8"
+        mock_connection.side_effect = [
+            {
+                "vol": vol_name,
+                "lun": 1,
+            },
+            {
+                "vol": vol_name,
+                "lun": 5,
+            },
+        ]
+        result = deepcopy(NVME_CONNECTION_INFO_AC_FILTERED)
+        self.driver._is_active_cluster_enabled = True
+        # Set up some CIDRs to block: this will block only one of the
+        # get four+three results back
+        self.driver.configuration.pure_nvme_cidr = NVME_CIDR_FILTERED
+        mock_secondary = mock.MagicMock()
+        self.driver._uniform_active_cluster_target_arrays = [mock_secondary]
+
+        real_result = self.driver.initialize_connection(vol, NVME_CONNECTOR)
+        self.assertDictEqual(result, real_result)
+        mock_get_nvme_ports.assert_has_calls(
+            [
+                mock.call(self.array),
+                mock.call(mock_secondary),
+            ]
+        )
+        mock_connection.assert_has_calls(
+            [
+                mock.call(self.array, vol_name, NVME_CONNECTOR),
+                mock.call(mock_secondary, vol_name, NVME_CONNECTOR),
+            ]
+        )
+
+    @mock.patch(NVME_DRIVER_OBJ + "._get_nguid")
+    @mock.patch(NVME_DRIVER_OBJ + "._get_wwn")
+    @mock.patch(NVME_DRIVER_OBJ + "._connect")
+    @mock.patch(NVME_DRIVER_OBJ + "._get_target_nvme_ports")
+    def test_initialize_connection_uniform_ac_cidrs(
+        self, mock_get_nvme_ports, mock_connection, mock_get_wwn,
+        mock_get_nguid
+    ):
+        repl_extra_specs = {
+            "replication_type": "<in> sync",
+            "replication_enabled": "<is> true",
+        }
+        vol, vol_name = self.new_fake_vol(type_extra_specs=repl_extra_specs)
+        mock_get_nvme_ports.side_effect = [NVME_PORTS, AC_NVME_PORTS]
+        mock_get_wwn.return_value = "3624a93709714b5cb91634c470002b2c8"
+        mock_get_nguid.return_value = "0009714b5cb916324a9374c470002b2c8"
+        mock_connection.side_effect = [
+            {
+                "vol": vol_name,
+                "lun": 1,
+            },
+            {
+                "vol": vol_name,
+                "lun": 5,
+            },
+        ]
+        result = deepcopy(NVME_CONNECTION_INFO_AC_FILTERED_LIST)
+
+        self.driver._is_active_cluster_enabled = True
+        # Set up some CIDRs to block: this will allow only 2 addresses from
+        # each host of the ActiveCluster, so we should check that we only
+        # get two+two results back
+        self.driver.configuration.pure_nvme = NVME_CIDR
+        self.driver.configuration.pure_nvme_cidr_list = NVME_CIDRS_FILTERED
+        mock_secondary = mock.MagicMock()
+        self.driver._uniform_active_cluster_target_arrays = [mock_secondary]
+
+        real_result = self.driver.initialize_connection(vol, NVME_CONNECTOR)
+        self.assertDictEqual(result, real_result)
+        mock_get_nvme_ports.assert_has_calls(
+            [
+                mock.call(self.array),
+                mock.call(mock_secondary),
+            ]
+        )
+        mock_connection.assert_has_calls(
+            [
+                mock.call(self.array, vol_name, NVME_CONNECTOR),
+                mock.call(mock_secondary, vol_name, NVME_CONNECTOR),
+            ]
+        )
+
     @mock.patch(NVME_DRIVER_OBJ + "._get_nguid")
     @mock.patch(NVME_DRIVER_OBJ + "._get_wwn")
     @mock.patch(NVME_DRIVER_OBJ + "._connect")

cinder/volume/drivers/pure.py

@@ -282,13 +282,6 @@ class PureBaseVolumeDriver(san.SanDriver):
                 ssl_cert_path = replication_device.get("ssl_cert_path", None)
                 repl_type = replication_device.get("type",
                                                    REPLICATION_TYPE_ASYNC)
-                if (
-                    repl_type == REPLICATION_TYPE_SYNC
-                    and "NVMe" in self._storage_protocol
-                ):
-                    msg = _('NVMe driver does not support synchronous '
-                            'replication')
-                    raise PureDriverException(reason=msg)
                 uniform = strutils.bool_from_string(
                     replication_device.get("uniform", False))
 

doc/source/configuration/block-storage/drivers/pure-storage-driver.rst

@@ -26,8 +26,6 @@ means you do not have the high-availability and non-disruptive upgrade
 benefits provided by FlashArray. Multipathing must be used to take advantage
 of these benefits.
 
-The NVMe driver does not support synchronous replication using ActiveCluster.
-
 Supported operations
 ~~~~~~~~~~~~~~~~~~~~
 
@@ -51,8 +49,7 @@ Supported operations
 
 * Create a thin provisioned volume.
 
-* Replicate volumes to remote Pure Storage array(s) - synchronous replication
+* Replicate volumes to remote Pure Storage array(s)
-  is not supported with the NVMe driver.
 
 QoS support for the Pure Storage drivers include the ability to set the
 following capabilities in the OpenStack Block Storage API
@@ -267,10 +264,6 @@ of the remote array.
 The ``REPLICATION_TYPE`` value for the ``type`` key can be either ``sync`` or
 ``async``
 
-.. note::
-
-   Synchronous replication is not supported by the NVMe driver.
-
 If the ``type`` is ``sync`` volumes will be created in a stretched Pod. This
 requires two arrays pre-configured with Active Cluster enabled. You can
 optionally specify ``uniform`` as ``true`` or ``false``, this will instruct

releasenotes/notes/pure-storage-nvme-driver-f4217c00379c4827.yaml

@@ -4,4 +4,4 @@ features:
     Pure Storage adds a new driver to support NVMe-RoCE for the
     FlashArray.
     All features of the iSCSI and FC drivers are fully supported by this new
-    driver with the exception of synchronous replication.
+    driver.