diff --git a/doc/source/_static/kernal_parameters.png b/doc/source/_static/kernal_parameters.png new file mode 100644 index 0000000..9706f66 Binary files /dev/null and b/doc/source/_static/kernal_parameters.png differ diff --git a/doc/source/appendix.rst b/doc/source/appendix.rst index 5eb50f6..782da29 100644 --- a/doc/source/appendix.rst +++ b/doc/source/appendix.rst @@ -7,6 +7,9 @@ Appendix Mellanox site where users can read about possible configurations: +- `Mellanox ConnectX-4 `_ - `Mellanox ConnectX-3 pro `_ - `HowTo Install Mirantis Fuel OpenStack with Mellanox `_ - `Mellanox InfiniBand Switches `_ +- `Mellanox NEO `_ + diff --git a/doc/source/conf.py b/doc/source/conf.py index a2fee16..bf9de10 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -47,7 +47,7 @@ source_suffix = '.rst' master_doc = 'index' # General information about the project. -project = u'The Mellanox ConnectX-3 pro plugin for Fuel' +project = u'The Mellanox ConnectX-4 plugin for Fuel' copyright = u'2016, mellanox' # The version info for the project you're documenting, acts as replacement for @@ -55,9 +55,9 @@ copyright = u'2016, mellanox' # built documents. # # The short X.Y version. -version = '3.0' +version = '3.2' # The full version, including alpha/beta/rc tags. -release = '3.0-3.0.0-1' +release = '3.2-3.2.0-1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -200,7 +200,7 @@ latex_elements = { # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'fuel-plugin-mellanox.tex', u'The Mellanox ConnectX-3 pro Plugin for Fuel Documentation', + ('index', 'fuel-plugin-mellanox.tex', u'The Mellanox ConnectX-4 Plugin for Fuel Documentation', u'Mellanox', 'manual'), ] @@ -238,7 +238,7 @@ latex_elements = { 'classoptions': ',openany,oneside', 'babel' : '\\usepackage[e # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'fuel-plugin-mellanox', u'Guide to the Mellanox ConnectX-3 pro plugin ver. 3.0-3.0.0-1 for Fuel 8', + ('index', 'fuel-plugin-mellanox', u'Guide to the Mellanox ConnectX-4 plugin ver. 3.2-3.2.0-1 for Fuel 8', [u'Mellanox'], 1) ] @@ -252,8 +252,8 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'fuel-plugin-mellanox', u'The Mellanox ConnectX-3 pro Plugin for Fuel Documentation', - u'Mellanox', 'fuel-plugin-mellanox', 'The Mellanox ConnectX-3 pro Plugin for Fuel Documentation', + ('index', 'fuel-plugin-mellanox', u'The Mellanox ConnectX-4 Plugin for Fuel Documentation', + u'Mellanox', 'fuel-plugin-mellanox', 'The Mellanox ConnectX-4 Plugin for Fuel Documentation', 'Miscellaneous'), ] @@ -269,7 +269,7 @@ texinfo_documents = [ # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False -# Insert footnotes where they are defined instead of +# Insert footnotes where they are defined instead of # at the end. pdf_inline_footnotes = True @@ -278,7 +278,7 @@ pdf_inline_footnotes = True # -- Options for Epub output ---------------------------------------------- # Bibliographic Dublin Core info. -epub_title = u'The Mellanox ConnectX-3 pro plugin' +epub_title = u'The Mellanox ConnectX-4 plugin' epub_author = u'mellanox' epub_publisher = u'Mellanox' epub_copyright = u'2016, Mellanox' diff --git a/doc/source/definitions.rst b/doc/source/definitions.rst index 88edde6..d136913 100644 --- a/doc/source/definitions.rst +++ b/doc/source/definitions.rst @@ -30,3 +30,9 @@ OpenSM PKey PKEY stands for partition key. It is a 16 bit field within the InfiniBand header called BTH (Base Transport Header). A collection of endnodes with the same PKey in their PKey Tables are referred to as being members of a partition. + +ConnectX-4 + `ConnectX-4 `_ adapter cards with Virtual Protocol Interconnect (VPI), supporting EDR 100Gb/s InfiniBand and 100Gb/s Ethernet connectivity, provide the highest performance and most flexible solution for high-performance, Web 2.0, Cloud, data analytics, database, and storage platforms. + +NEO + Mellanox NEO™ is a powerful platform for managing scale-out computing networks. Mellanox NEO™ enables data center operators to efficiently provision, monitor and operate the modern data center fabric. diff --git a/doc/source/guide.rst b/doc/source/guide.rst index 9ceea46..9bed5b1 100644 --- a/doc/source/guide.rst +++ b/doc/source/guide.rst @@ -7,7 +7,8 @@ Mellanox plugin configuration If you plan to enable VM to VM RDMA and to use iSER storage transport you need to configure switching fabric to support the features. -**Ethernet network:** +Ethernet network: +----------------- #. Configure the required VLANs and enable flow control on the Ethernet switch ports. #. All related VLANs should be enabled on the Mellanox switch ports (for relevant Fuel logical networks). @@ -47,27 +48,47 @@ If you plan to enable VM to VM RDMA and to use iSER storage transport you need t ... -**Infiniband network:** -If you use OpenSM you need to enable virtualization and allow all PKeys: +Infiniband network: +------------------- -#. Create a new opensm.conf file:: +Mellanox **UFM** is a pre-requisite for using the Mellanox plugin for Fuel 8.0 with InfiniBand fabrics. Mellanox.s Unified Fabric Manager (UFM®) is a powerful platform for managing scale-out computing environments. UFM enables data center operators to monitor, efficiently provision, and operate the modern data center fabric. UFM is licensed per managed fabric node. For more information on how to obtain UFM, please visit Mellanox.com. - opensm -c /etc/opensm/opensm.conif +Update OpenSM configurations on UFM node as follows: -#. Enable virtualization by editing /etc/opensm/opensm.conf and changing the allow_both_pkeys value to TRUE.:: +#. Update opensm.conf file and make sure of the following:: - allow_both_pkeys TRUE + vim /opt/ufm/conf/opensm/opensm.conf + - virt_enabled 2 + - no_partition_enforcement TRUE + - part_enforce off + - allow_both_pkeys FALSE -#. Define the partition keys which are analog for Ethernet VLAN. Each VLAN will be mapped to one PK. Add/Change the following with the command :: +#. Update the partitions.conf file:: - vi /etc/opensm/partitions.conf file: - (Example) - management=0x7fff,ipoib, sl=0, defmember=full : ALL, ALL_SWITCHES=full,SELF=full; - vlan1=0x1, ipoib, sl=0, defmember=full : ALL; - vlan2=0x2, ipoib, sl=0, defmember=full : ALL; - . . . - vlan100=0x64, ipoib, sl=0, defmember=full : ALL; + vim /opt/ufm/conf/partitions.conf.user_ext -#. Restart OpenSM:: + Example: - /etc/init.d/opensmd restart + vlan1=0x1, ipoib, sl=0, defmember=full: ALL_CAS; + vlan2=0x2, ipoib, sl=0, defmember=full: ALL_CAS; + vlan3=0x3, ipoib, sl=0, defmember=full: ALL_CAS; + + vlan4=0x4, ipoib, sl=0, defmember=full: SELF; + vlan5=0x5, ipoib, sl=0, defmember=full: SELF; + vlan6=0x6, ipoib, sl=0, defmember=full: SELF; + vlan7=0x7, ipoib, sl=0, defmember=full: SELF; + vlan8=0x8, ipoib, sl=0, defmember=full: SELF; + vlan9=0x9, ipoib, sl=0, defmember=full: SELF; + . . . + vlan20=0x14, ipoib, sl=0, defmember=full: SELF; + + **Note**: In this example + + - Infra networks VLANs are 1-3 so VLAN2 is assigned to PK 0x2 and will be used for Openstack Management network and VLAN3 is assigned to PK 0x3 and will be used for Openstack Storage network. + - Private VLANs are 4-20 so VLANs 4 through 20 are assigned to PKs 0x4 to 0x14 will be used for Tenant networks. + - VLAN1 is defined, but not used for consistency with Ethernet setup installation. + - The maximum number of VLANs is 128. + +#. Restart ufmd:: + + /etc/init.d/ufmd restart diff --git a/doc/source/index.rst b/doc/source/index.rst index c8f547c..e0d893a 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -4,7 +4,7 @@ contain the root `toctree` directive. ===================================================================== -Guide to the Mellanox ConnectX-3 pro Plugin ver. 3.0-3.0.0-1 for Fuel +Guide to the Mellanox ConnectX-4 Plugin ver. 3.2-3.2.0-1 for Fuel ===================================================================== User documentation @@ -15,6 +15,7 @@ User documentation definitions overview + release_notes guide installation post_deployment diff --git a/doc/source/installation.rst b/doc/source/installation.rst index c98ea52..681e14e 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -11,22 +11,52 @@ To install Mellanox plugin, follow these steps: #. Download the plugin rpm file for MOS 8.0 from `Fuel Plugin Catalog `_. #. Copy the plugin on already installed Fuel Master. scp can be used for that.:: - # scp mellanox-plugin-3.0-3.0.0-1.noarch.rpm root@:/tmp + # scp mellanox-plugin-3.2-3.2.0-1.noarch.rpm root@:/tmp + #. Install the plugin:: # cd /tmp - # fuel plugins --install mellanox-plugin-3.0-3.0.0-1.noarch.rpm + # fuel plugins --install mellanox-plugin-3.2-3.2.0-1.noarch.rpm #. Verify the plugin was installed successfully by having it listed using ``fuel plugins`` command:: # fuel plugins # id | name | version | package_version # ---|-------------------|---------|---------------- - # 1 | mellanox-plugin | 3.0.0 | 3.0.0 + # 1 | mellanox-plugin | 3.2.0 | 3.0.0 -#. Create new bootstrap image for supporting infiniband networks (``create_mellanox_vpi_bootstrap can be used``)::: +#. Define bootstrap discovery parameters to be burnt on Mellanox Adapters cards: + + - **link_type** , available link_type values are: + + - ``eth`` for changing link type to Ethernet + - ``ib`` for changing link type to Infiniband + - ``current`` for leaving link type as is + + - **max_num_vfs** as integer, default is set to 16. + +#. Create Bootstrap discovery image for detecting Mellanox HW and support related configurations + with pre-defined parameters:: + + [root@fuel ~]# create_mellanox__bootstrap --link_type $link_type --max_num_vfs $max_num_vfs + [root@fuel ~]# create_mellanox_bootstrap --help + + :: + + usage: create_mellanox_bootstrap [-h] [--link_type {eth,ib,current}] + [--max_num_vfs MAX_NUM_VFS] + Available link_type values are: + ------------------------------- + - eth for changing link type to Ethernet + - ib for changing link type to Infiniband + - current for leaving link type as is + + optional arguments: + -h, --help show this help message and exit + --link_type {eth,ib,current} + --max_num_vfs MAX_NUM_VFS + an integer for the maximum number of vfs to be burned in bootstrap - [root@fuel ~]# create_mellanox_vpi_bootstrap :: @@ -39,8 +69,24 @@ To install Mellanox plugin, follow these steps: . . . Bootstrap image f790e9f8-5bc5-4e61-9935-0640f2eed949 has been activated. -#. In case of using the customized bootstrap image, you must reboot your target nodes with the new bootstrap image you just created. - If you already have discovered nodes you can either reboot them manually or use :bash: `reboot_bootstrap_nodes` command. Run :bash: `reboot_bootstrap_nodes -h` for help. +#. Reboot nodes after installing plugin:: + + [root@fuel ~]# reboot_bootstrap_nodes -a + [root@fuel ~]# reboot_bootstrap_nodes -h + + :: + + Usage: reboot_bootstrap_nodes [-e environment_id] [-h] [-a] + This script is used to trigger reboot for nodes in 'discover' status, + of a given environment (if given) or of all environments. + Please wait for nodes to boot again after triggering this script. + + Options: + + -h Display the help message. + -e Reboot all nodes in state 'discover' of the given environment. + -a Reboot all nodes in state 'discover' of all environments. + #. Create an environment - for more information please see `how to create an environment `_. We support both main network configurations: @@ -51,6 +97,12 @@ To install Mellanox plugin, follow these steps: .. image:: ./_static/ml2_driver.png .. :alt: Network Configuration Type +#. Adjust the kernal parameters in the settings tab which is a condition for both iSER and SRIOV. + Open the Settings tab, select General section and then add ``intel_iommu=on`` at the beginning of the initial parameters. + + .. image:: ./_static/kernal_parameters.png + .. :alt: Hypervisor Type + #. Enable KVM hypervisor type. KVM is required to enable Mellanox Openstack features. Open the Settings tab, select Compute section and then choose KVM hypervisor type. @@ -79,16 +131,6 @@ To install Mellanox plugin, follow these steps: .. image:: ./_static/sriov.png .. :alt: Enable SR-IOV - #. Support quality of service over VLAN networks with Mellanox SR-IOV direct ports (Neutron) - **Note**: Relevant for `VLAN segmentation` only - If selected, Neutron "Quality of service" (QoS) will be enabled for VLAN networks and ports over Mellanox HCAs. - **Note**: This feature is supported only if: - - - Ethernet mode is used - - SR-IOV is enabled - - .. image:: ./_static/qos.png - .. :alt: Enable QoS #. Support NEO SDN controller auto VLAN Provisioning (Neutron) **Note**: Relevant for `VLAN segmentation` only @@ -102,13 +144,6 @@ To install Mellanox plugin, follow these steps: Additional info about NEO can be found by link: https://community.mellanox.com/docs/DOC-2155 - #. Support VXLAN Offloading (Neutron) - **Note**: Relevant for `tunneling segmentation` only - - If selected, Mellanox hardware will be used to achieve a better performance and significant CPU overhead reduction using VXLAN traffic offloading. - - .. image:: ./_static/vxlan.png - .. :alt: Enable VXLAN offloading #. iSER protocol for volumes (Cinder) **Note**: Relevant for both `VLAN segmentation` and `VLAN segmentation` deployments @@ -132,7 +167,7 @@ To install Mellanox plugin, follow these steps: #. In Ethernet cloud, when using SR-IOV & iSER, one of the virtual NICs for SR-IOV will be reserved to the storage network. -#. When using SR-IOV you can set the number of virtual NICs (virtual functions) to up to 62 +#. When using SR-IOV you can set the number of virtual NICs (virtual functions) to up to 31 if your hardware and system capabilities like memory and BIOS support it). - In any case of SR-IOV hardware limitation, the installation will try to fallback a VF number to the default of 8 VFs. + In any case of SR-IOV hardware limitation, the installation will try to fallback a VF number to the default of 16 VFs. diff --git a/doc/source/known_issues.rst b/doc/source/known_issues.rst index f861d27..df7c1c2 100644 --- a/doc/source/known_issues.rst +++ b/doc/source/known_issues.rst @@ -6,11 +6,11 @@ Known issues ============ Issue 1 - - Description: For custom (OEM) adapter cards based on Mellanox ConnectX-3 / ConnectX-3 Pro ICs, adapter firmware must be manually burnt prior to the installation with SR-IOV support + - Description: For custom (OEM) adapter cards based on Mellanox ConnectX-4 ICs, adapter firmware must be manually burnt prior to the installation with SR-IOV support - Workaround: See `the firmware installation instructions `_. Issue 2 - - Description: The number of SR-IOV virtual functions supported by Mellanox adapters is up to 16 on ConnectX-3 adapters and up to 62 on ConnectX-3 Pro adapters (depends on your HW capabilities). + - Description: The number of SR-IOV virtual functions supported by Mellanox adapters is up to 31 on ConnectX-4 adapters (depends on your HW capabilities). - Workaround: NA Issue 3 @@ -18,21 +18,21 @@ Issue 3 - Workaround: NA Issue 4 - - Description: A single port HCA might not be supported for SRIOV and iSER over Ethernet network. - - Workaround: NA + - Description: Changing port type in bootstrap stage over a single port HCA is not supported + - Workaround: Create a bootstrap image with link type current, and change the port type manually. Issue 5 - - Description: SR-IOV QoS is supported only with updating SR-IOV existing ports with a policy. QoS-policy detach might result in non accurate bandwidth limit. (https://bugs.launchpad.net/neutron/+bug/1504165). - - Workaround: Delete port / instance and attach a new port. - -Issue 6 - Description: Starting large amount (>15) of IB VMs with normal port at once may result in some VMs not getting DHCP over InfiniBand networks. - Workaround: Reboot VMs that didn't get IP from DHCP on time or start VMs in smaller chunks (<10). -Issue 7 - - Description: After large InfinBand deployment of more than ~20 nodes at once with Controllers HA, it might take time for controllers services to stabilize. - - Workaround: Restart openibd service on controller nodes after the deployment, or deploy with phases. - -Issue 8 +Issue 6 - Description: Network verification for IB network is not supported over untagged networks or after deployment. - Workaround: NA + +Issue 7 + - Description: When using NEO auto provisioning, network verification should fail for the private network VLANs + - Workaround: NA + +Issue 8 + - Description: When deploying an Infiniband cluster with iSER over VLAN, all controllers should be deployed at once. + - Workaround: Use untagged storage network when using Infiniband with iSER over VLAN, or deploy all controllers at once. diff --git a/doc/source/overview.rst b/doc/source/overview.rst index 56faec1..591db5c 100644 --- a/doc/source/overview.rst +++ b/doc/source/overview.rst @@ -8,8 +8,8 @@ Mellanox plugin | The Mellanox Fuel plugin is a bundle of scripts, packages and metadata that will extend Fuel and add Mellanox features such as SR-IOV for networking and iSER protocol for storage. -| Fuel can configure `Mellanox ConnectX-3 Pro - `_ +| Fuel can configure `Mellanox ConnectX-4 + `_ network adapters to accelerate the performance of compute and storage traffic. This implements the following performance enhancements: @@ -42,17 +42,8 @@ Requirements | Mirantis OpenStack compatibility | 8.0 | +-----------------------------------+-----------------+ -| The Mellanox ConnectX-3 Pro adapters family supports up to 40/56 Gb. To reach 56 Gb speed in - your network with ConnectX-3 Pro adapters, you must use Mellanox Ethernet / Infiniband switches - supporting 56 Gb (e.g. SX1710, SX6710). The switch ports should be configured specifically to use - 56 Gb speed. No additional configuration is required on the adapter side. For additional - information about how to run in 56GbE speed, see `HowTo Configure 56GbE Link on Mellanox Adapters - and Switches `_. +| The Mellanox ConnectX-4 adapters family supports up to 100 Gb/s. To reach 100 Gb/s speed in your + network with ConnectX-4 adapters, you must use Mellanox Ethernet / Infiniband switches supporting 100 Gb + (e.g. SN2700 (ETH), SB7700 (IB)). The switch ports should be configured specifically to use 100 Gb speed. No + additional configuration is required on the adapter side. -Limitations ------------ - -- Mellanox SR-IOV is supported only when choosing Neutron with VLAN segmentation. -- ConnectX-3 Pro adapters are required in order to enable VXLAN HW offload over Ethernet networks. -- QoS feature is implemented only for Ethernet VLAN SR-IOV ports using ConnectX-3 Pro adapters. -- Infiniband is configured by using OpenSM only. diff --git a/doc/source/release_notes.rst b/doc/source/release_notes.rst new file mode 100644 index 0000000..9448b7a --- /dev/null +++ b/doc/source/release_notes.rst @@ -0,0 +1,7 @@ +.. raw:: pdf + + PageBreak + +Release notes +============= + diff --git a/doc/source/supported_images.rst b/doc/source/supported_images.rst index bc07319..49bec3c 100644 --- a/doc/source/supported_images.rst +++ b/doc/source/supported_images.rst @@ -8,11 +8,11 @@ Supported images +-------+--------------------+---------------------------------+ | Issue | Supported OS | Tested kernel | +=======+====================+=================================+ -| 1 | CentOS7 | 3.10.0-229.14.1.el7.x86_64 | +| 1 | CentOS7 | 3.10.0-327.13.1.e17.x86_64 | +-------+--------------------+---------------------------------+ -| 2 | ubuntu14.04 | 3.13.0-67-generic | +| 2 | ubuntu14.04 | 3.13.0-85-generic | +-------+--------------------+---------------------------------+ | 3 | Cirros Mellanox | 3.11.0-26-generic | +-------+--------------------+---------------------------------+ -This Fuel Mellanox plugin ver. 3.0-3.0.0-1 is using MLNX_OFED version 3.1-1.5.5. +This Fuel Mellanox plugin ver. 3.2-3.2.0-1 is using MLNX_OFED_LINUX version 3.3-1.5.0. diff --git a/metadata.yaml b/metadata.yaml index 1e92dd0..8a5fe11 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -5,7 +5,7 @@ name: mellanox-plugin title: Mellanox ConnectX-4 Openstack Features # Plugin version -version: 3.1.18 +version: 3.2.0 # Description description: Enable features over Mellanox ConnectX-4 Adapters