Adjust os-brick to support FCP on System z systems

There are some platform specific changes needed to allow os-brick to
support FCP on System z: the System z specific format of the device
file paths, adding and removing devices explicitly, and ignoring vHBAs
which are offline.

Based on Cinder: https://review.openstack.org/#/c/149256/

Partial-Implements blueprint linux-systemz

Change-Id: Ie75454da974939e03dfc77e760ba5215324b7fdc

os_brick/initiator/connector.py

@@ -22,11 +22,16 @@ each of the supported transport protocols.
 
 import copy
 import os
+import platform
 import socket
 import time
 
 from oslo_concurrency import lockutils
 from oslo_concurrency import processutils as putils
+import six
+
+S390X = "s390x"
+S390 = "s390"
 
 from os_brick import exception
 from os_brick import executor
@@ -112,9 +117,10 @@ class InitiatorConnector(executor.Executor):
     def factory(protocol, root_helper, driver=None,
                 execute=putils.execute, use_multipath=False,
                 device_scan_attempts=DEVICE_SCAN_ATTEMPTS_DEFAULT,
+                arch=platform.machine(),
                 *args, **kwargs):
-        """Build a Connector object based upon protocol."""
-        LOG.debug("Factory for %s" % protocol)
+        """Build a Connector object based upon protocol and architecture."""
+        LOG.debug("Factory for %s on %s" % (protocol, arch))
         protocol = protocol.upper()
         if protocol == "ISCSI":
             return ISCSIConnector(root_helper=root_helper,
@@ -131,13 +137,22 @@ class InitiatorConnector(executor.Executor):
                                  device_scan_attempts=device_scan_attempts,
                                  *args, **kwargs)
         elif protocol == "FIBRE_CHANNEL":
-            return FibreChannelConnector(root_helper=root_helper,
-                                         driver=driver,
-                                         execute=execute,
-                                         use_multipath=use_multipath,
-                                         device_scan_attempts=
-                                         device_scan_attempts,
-                                         *args, **kwargs)
+            if arch in (S390, S390X):
+                return FibreChannelConnectorS390X(root_helper=root_helper,
+                                                  driver=driver,
+                                                  execute=execute,
+                                                  use_multipath=use_multipath,
+                                                  device_scan_attempts=
+                                                  device_scan_attempts,
+                                                  *args, **kwargs)
+            else:
+                return FibreChannelConnector(root_helper=root_helper,
+                                             driver=driver,
+                                             execute=execute,
+                                             use_multipath=use_multipath,
+                                             device_scan_attempts=
+                                             device_scan_attempts,
+                                             *args, **kwargs)
         elif protocol == "AOE":
             return AoEConnector(root_helper=root_helper,
                                 driver=driver,
@@ -680,31 +695,12 @@ class FibreChannelConnector(InitiatorConnector):
         LOG.debug("execute = %s" % self._execute)
         device_info = {'type': 'block'}
 
+        hbas = self._linuxfc.get_fc_hbas_info()
         ports = connection_properties['target_wwn']
-        wwns = []
-        # we support a list of wwns or a single wwn
-        if isinstance(ports, list):
-            for wwn in ports:
-                wwns.append(str(wwn))
-        elif isinstance(ports, basestring):
-            wwns.append(str(ports))
+        possible_devs = self._get_possible_devices(hbas, ports)
 
-        # We need to look for wwns on every hba
-        # because we don't know ahead of time
-        # where they will show up.
-        hbas = self._linuxfc.get_fc_hbas_info()
-        host_devices = []
-        for hba in hbas:
-            pci_num = self._get_pci_num(hba)
-            if pci_num is not None:
-                for wwn in wwns:
-                    target_wwn = "0x%s" % wwn.lower()
-                    host_device = ("/dev/disk/by-path/pci-%s-fc-%s-lun-%s" %
-                                   (pci_num,
-                                    target_wwn,
-                                    connection_properties.get(
-                                        'target_lun', 0)))
-                    host_devices.append(host_device)
+        lun = connection_properties.get('target_lun', 0)
+        host_devices = self._get_host_devices(possible_devs, lun)
 
         if len(host_devices) == 0:
             # this is empty because we don't have any FC HBAs
@@ -777,6 +773,50 @@ class FibreChannelConnector(InitiatorConnector):
         device_info['devices'] = devices
         return device_info
 
+    def _get_host_devices(self, possible_devs, lun):
+        host_devices = []
+        for pci_num, target_wwn in possible_devs:
+            host_device = "/dev/disk/by-path/pci-%s-fc-%s-lun-%s" % (
+                pci_num,
+                target_wwn,
+                lun)
+            host_devices.append(host_device)
+        return host_devices
+
+    def _get_possible_devices(self, hbas, wwnports):
+        """Compute the possible valid fibre channel device options.
+
+        :param hbas: available hba devices.
+        :param wwnports: possible wwn addresses. Can either be string
+        or list of strings.
+
+        :returns: list of (pci_id, wwn) tuples
+
+        Given one or more wwn (mac addresses for fibre channel) ports
+        do the matrix math to figure out a set of pci device, wwn
+        tuples that are potentially valid (they won't all be). This
+        provides a search space for the device connection.
+
+        """
+        # the wwn (think mac addresses for fiber channel devices) can
+        # either be a single value or a list. Normalize it to a list
+        # for further operations.
+        wwns = []
+        if isinstance(wwnports, list):
+            for wwn in wwnports:
+                wwns.append(str(wwn))
+        elif isinstance(wwnports, six.string_types):
+            wwns.append(str(wwnports))
+
+        raw_devices = []
+        for hba in hbas:
+            pci_num = self._get_pci_num(hba)
+            if pci_num is not None:
+                for wwn in wwns:
+                    target_wwn = "0x%s" % wwn.lower()
+                    raw_devices.append((pci_num, target_wwn))
+        return raw_devices
+
     @synchronized('connect_volume')
     def disconnect_volume(self, connection_properties, device_info):
         """Detach the volume from instance_name.
@@ -797,6 +837,9 @@ class FibreChannelConnector(InitiatorConnector):
             LOG.debug("devices to remove = %s" % devices)
             self._linuxscsi.flush_multipath_device(multipath_id)
 
+        self._remove_devices(connection_properties, devices)
+
+    def _remove_devices(self, connection_properties, devices):
         # There may have been more than 1 device mounted
         # by the kernel for this volume.  We have to remove
         # all of them
@@ -825,6 +868,69 @@ class FibreChannelConnector(InitiatorConnector):
         return pci_num
 
 
+class FibreChannelConnectorS390X(FibreChannelConnector):
+    """Connector class to attach/detach Fibre Channel volumes on S390X arch."""
+
+    def __init__(self, root_helper, driver=None,
+                 execute=putils.execute, use_multipath=False,
+                 device_scan_attempts=DEVICE_SCAN_ATTEMPTS_DEFAULT,
+                 *args, **kwargs):
+        super(FibreChannelConnectorS390X, self).__init__(root_helper,
+                                                         driver=driver,
+                                                         execute=execute,
+                                                         device_scan_attempts=
+                                                         device_scan_attempts,
+                                                         *args, **kwargs)
+        LOG.debug("Initializing Fibre Channel connector for S390")
+        self._linuxscsi = linuxscsi.LinuxSCSI(root_helper, execute)
+        self._linuxfc = linuxfc.LinuxFibreChannelS390X(root_helper, execute)
+        self.use_multipath = use_multipath
+
+    def set_execute(self, execute):
+        super(FibreChannelConnectorS390X, self).set_execute(execute)
+        self._linuxscsi.set_execute(execute)
+        self._linuxfc.set_execute(execute)
+
+    def _get_host_devices(self, possible_devs, lun):
+        host_devices = []
+        for pci_num, target_wwn in possible_devs:
+            target_lun = self._get_lun_string(lun)
+            host_device = self._get_device_file_path(
+                pci_num,
+                target_wwn,
+                target_lun)
+            self._linuxfc.configure_scsi_device(pci_num, target_wwn,
+                                                target_lun)
+            host_devices.append(host_device)
+        return host_devices
+
+    def _get_lun_string(self, lun):
+        target_lun = 0
+        if lun < 256:
+            target_lun = "0x00%02x000000000000" % lun
+        elif lun <= 0xffffffff:
+            target_lun = "0x%08x00000000" % lun
+        return target_lun
+
+    def _get_device_file_path(self, pci_num, target_wwn, target_lun):
+        host_device = "/dev/disk/by-path/ccw-%s-zfcp-%s:%s" % (
+            pci_num,
+            target_wwn,
+            target_lun)
+        return host_device
+
+    def _remove_devices(self, connection_properties, devices):
+        hbas = self._linuxfc.get_fc_hbas_info()
+        ports = connection_properties['target_wwn']
+        possible_devs = self._get_possible_devices(hbas, ports)
+        lun = connection_properties.get('target_lun', 0)
+        target_lun = self._get_lun_string(lun)
+        for pci_num, target_wwn in possible_devs:
+            self._linuxfc.deconfigure_scsi_device(pci_num,
+                                                  target_wwn,
+                                                  target_lun)
+
+
 class AoEConnector(InitiatorConnector):
     """Connector class to attach/detach AoE volumes."""
     def __init__(self, root_helper, driver=None,

os_brick/initiator/linuxfc.py

@@ -138,3 +138,75 @@ class LinuxFibreChannel(linuxscsi.LinuxSCSI):
                     wwnns.append(wwnn)
 
         return wwnns
+
+
+class LinuxFibreChannelS390X(LinuxFibreChannel):
+    def __init__(self, root_helper, execute=putils.execute,
+                 *args, **kwargs):
+        super(LinuxFibreChannelS390X, self).__init__(root_helper, execute,
+                                                     *args, **kwargs)
+
+    def get_fc_hbas_info(self):
+        """Get Fibre Channel WWNs and device paths from the system, if any."""
+
+        hbas = self.get_fc_hbas()
+        if not hbas:
+            return []
+
+        hbas_info = []
+        for hba in hbas:
+            if hba['port_state'] == 'Online':
+                wwpn = hba['port_name'].replace('0x', '')
+                wwnn = hba['node_name'].replace('0x', '')
+                device_path = hba['ClassDevicepath']
+                device = hba['ClassDevice']
+                hbas_info.append({'port_name': wwpn,
+                                  'node_name': wwnn,
+                                  'host_device': device,
+                                  'device_path': device_path})
+        return hbas_info
+
+    def configure_scsi_device(self, device_number, target_wwn, lun):
+        """Write the LUN to the port's unit_add attribute.
+
+        If auto-discovery of LUNs is disabled on s390 platforms
+        luns need to be added to the configuration through the
+        unit_add interface
+        """
+        LOG.debug("Configure lun for s390: device_number=(%(device_num)s) "
+                  "target_wwn=(%(target_wwn)s) target_lun=(%(target_lun)s)",
+                  {'device_num': device_number,
+                   'target_wwn': target_wwn,
+                   'target_lun': lun})
+        zfcp_device_command = ("/sys/bus/ccw/drivers/zfcp/%s/%s/unit_add" %
+                               (device_number, target_wwn))
+        LOG.debug("unit_add call for s390 execute: %s", zfcp_device_command)
+        try:
+            self.echo_scsi_command(zfcp_device_command, lun)
+        except putils.ProcessExecutionError as exc:
+            msg = _LW("unit_add call for s390 failed exit (%(code)s), "
+                      "stderr (%(stderr)s)")
+            LOG.warn(msg, {'code': exc.exit_code, 'stderr': exc.stderr})
+
+    def deconfigure_scsi_device(self, device_number, target_wwn, lun):
+        """Write the LUN to the port's unit_remove attribute.
+
+        If auto-discovery of LUNs is disabled on s390 platforms
+        luns need to be removed from the configuration through the
+        unit_remove interface
+        """
+        LOG.debug("Deconfigure lun for s390: "
+                  "device_number=(%(device_num)s) "
+                  "target_wwn=(%(target_wwn)s) target_lun=(%(target_lun)s)",
+                  {'device_num': device_number,
+                   'target_wwn': target_wwn,
+                   'target_lun': lun})
+        zfcp_device_command = ("/sys/bus/ccw/drivers/zfcp/%s/%s/unit_remove" %
+                               (device_number, target_wwn))
+        LOG.debug("unit_remove call for s390 execute: %s", zfcp_device_command)
+        try:
+            self.echo_scsi_command(zfcp_device_command, lun)
+        except putils.ProcessExecutionError as exc:
+            msg = _LW("unit_remove call for s390 failed exit (%(code)s), "
+                      "stderr (%(stderr)s)")
+            LOG.warn(msg, {'code': exc.exit_code, 'stderr': exc.stderr})

os_brick/tests/initiator/test_connector.py

@@ -113,6 +113,10 @@ class ConnectorTestCase(base.TestCase):
         obj = connector.InitiatorConnector.factory('fibre_channel', None)
         self.assertEqual(obj.__class__.__name__, "FibreChannelConnector")
 
+        obj = connector.InitiatorConnector.factory('fibre_channel', None,
+                                                   arch='s390x')
+        self.assertEqual(obj.__class__.__name__, "FibreChannelConnectorS390X")
+
         obj = connector.InitiatorConnector.factory('aoe', None)
         self.assertEqual(obj.__class__.__name__, "AoEConnector")
 
@@ -794,6 +798,45 @@ class FibreChannelConnectorTestCase(ConnectorTestCase):
                           connection_info['data'])
 
 
+class FibreChannelConnectorS390XTestCase(ConnectorTestCase):
+
+    def setUp(self):
+        super(FibreChannelConnectorS390XTestCase, self).setUp()
+        self.connector = connector.FibreChannelConnectorS390X(
+            None, execute=self.fake_execute, use_multipath=False)
+        self.assertIsNotNone(self.connector)
+        self.assertIsNotNone(self.connector._linuxfc)
+        self.assertEqual(self.connector._linuxfc.__class__.__name__,
+                         "LinuxFibreChannelS390X")
+        self.assertIsNotNone(self.connector._linuxscsi)
+
+    @mock.patch.object(linuxfc.LinuxFibreChannelS390X, 'configure_scsi_device')
+    def test_get_host_devices(self, mock_configure_scsi_device):
+        lun = 2
+        possible_devs = [(3, 5), ]
+        devices = self.connector._get_host_devices(possible_devs, lun)
+        mock_configure_scsi_device.assert_called_with(3, 5,
+                                                      "0x0002000000000000")
+        self.assertEqual(1, len(devices))
+        device_path = "/dev/disk/by-path/ccw-3-zfcp-5:0x0002000000000000"
+        self.assertEqual(devices[0], device_path)
+
+    @mock.patch.object(connector.FibreChannelConnectorS390X,
+                       '_get_possible_devices', return_value=[(3, 5), ])
+    @mock.patch.object(linuxfc.LinuxFibreChannelS390X, 'get_fc_hbas_info',
+                       return_value=[])
+    @mock.patch.object(linuxfc.LinuxFibreChannelS390X,
+                       'deconfigure_scsi_device')
+    def test_remove_devices(self, mock_deconfigure_scsi_device,
+                            mock_get_fc_hbas_info, mock_get_possible_devices):
+        connection_properties = {'target_wwn': 5, 'target_lun': 2}
+        self.connector._remove_devices(connection_properties, devices=None)
+        mock_deconfigure_scsi_device.assert_called_with(3, 5,
+                                                        "0x0002000000000000")
+        mock_get_fc_hbas_info.assert_called_once_with()
+        mock_get_possible_devices.assert_called_once_with([], 5)
+
+
 class FakeFixedIntervalLoopingCall(object):
     def __init__(self, f=None, *args, **kw):
         self.args = args

os_brick/tests/initiator/test_linuxfc.py

@@ -174,3 +174,76 @@ Class = "fc_host"
 
 
 """
+
+
+class LinuxFCS390XTestCase(LinuxFCTestCase):
+
+    def setUp(self):
+        super(LinuxFCS390XTestCase, self).setUp()
+        self.cmds = []
+        self.lfc = linuxfc.LinuxFibreChannelS390X(None,
+                                                  execute=self.fake_execute)
+
+    def test_get_fc_hbas_info(self):
+        def fake_exec(a, b, c, d, run_as_root=True, root_helper='sudo'):
+            return SYSTOOL_FC_S390X, None
+        self.lfc._execute = fake_exec
+        hbas_info = self.lfc.get_fc_hbas_info()
+        expected = [{'device_path': '/sys/devices/css0/0.0.02ea/'
+                                    '0.0.3080/host0/fc_host/host0',
+                     'host_device': 'host0',
+                     'node_name': '1234567898765432',
+                     'port_name': 'c05076ffe680a960'}]
+        self.assertEqual(expected, hbas_info)
+
+    def test_configure_scsi_device(self):
+        device_number = "0.0.2319"
+        target_wwn = "0x50014380242b9751"
+        lun = 1
+        self.lfc.configure_scsi_device(device_number, target_wwn, lun)
+        expected_commands = [('tee -a /sys/bus/ccw/drivers/zfcp/'
+                              '0.0.2319/0x50014380242b9751/unit_add')]
+        self.assertEqual(expected_commands, self.cmds)
+
+    def test_deconfigure_scsi_device(self):
+        device_number = "0.0.2319"
+        target_wwn = "0x50014380242b9751"
+        lun = 1
+        self.lfc.deconfigure_scsi_device(device_number, target_wwn, lun)
+        expected_commands = [('tee -a /sys/bus/ccw/drivers/zfcp/'
+                              '0.0.2319/0x50014380242b9751/unit_remove')]
+        self.assertEqual(expected_commands, self.cmds)
+
+SYSTOOL_FC_S390X = """
+Class = "fc_host"
+
+  Class Device = "host0"
+  Class Device path = "/sys/devices/css0/0.0.02ea/0.0.3080/host0/fc_host/host0"
+    active_fc4s         = "0x00 0x00 0x01 0x00 0x00 0x00 0x00 0x00 0x00 0x00 \
+    0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 \
+    0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 "
+    dev_loss_tmo        = "60"
+    maxframe_size       = "2112 bytes"
+    node_name           = "0x1234567898765432"
+    permanent_port_name = "0xc05076ffe6803081"
+    port_id             = "0x010014"
+    port_name           = "0xc05076ffe680a960"
+    port_state          = "Online"
+    port_type           = "NPIV VPORT"
+    serial_number       = "IBM00000000000P30"
+    speed               = "8 Gbit"
+    supported_classes   = "Class 2, Class 3"
+    supported_fc4s      = "0x00 0x00 0x01 0x00 0x00 0x00 0x00 0x00 0x00 0x00 \
+    0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 \
+    0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 "
+    supported_speeds    = "2 Gbit, 4 Gbit"
+    symbolic_name       = "IBM     2827            00000000000P30  \
+    PCHID: 0308 NPIV UlpId: 01EA0A00   DEVNO: 0.0.1234 NAME: dummy"
+    tgtid_bind_type     = "wwpn (World Wide Port Name)"
+    uevent              =
+
+    Device = "host0"
+    Device path = "/sys/devices/css0/0.0.02ea/0.0.3080/host0"
+      uevent              = "DEVTYPE=scsi_host"
+
+"""