Add support for switchdev mode in SR-IOV

In Kernel 4.10 supports changing SR-IOV to switchdev mode.
This mode allows to create VFs represontors which can manage
the SR-IOV VFs from the hypervsior.

This patch extends the tripleo::host::sriov::number_of_vf to
<physical_network>:<number_of_vfs>:<sriov_mode>,
where sriov_mode accepts legacy or switchdev.
if sriov_mode is not specified we default to legacy.

Change-Id: I578f956f2a8c6ee29a9d1ff38ee51765bcab05c1

lib/puppet/provider/sriov_vf_config/numvfs.rb

@@ -11,6 +11,7 @@ Puppet::Type.type(:sriov_vf_config).provide(:numvfs) do
   def create
     if File.file?(sriov_numvfs_path)
       _set_numvfs
+      _apply_hw_offload
     else
       warning("#{sriov_numvfs_path} doesn't exist. Check if #{sriov_get_interface} is a valid network interface supporting SR-IOV")
     end
@@ -42,16 +43,59 @@ Puppet::Type.type(:sriov_vf_config).provide(:numvfs) do
     File.write(sriov_numvfs_path,sriov_numvfs_value)
   end
 
+  def _apply_hw_offload
+    # Changing the mode of virtual functions to hw-offload
+    if ovs_mode == "switchdev"
+      cur_value = File.read(vendor_path).strip
+      if cur_value == "0x15b3"
+        vfs_pcis = get_vfs_pcis
+        # Unbinding virtual functions
+        vfs_pcis.each do|vfs_pci|
+          File.write("/sys/bus/pci/drivers/mlx5_core/unbind",vfs_pci)
+        end
+      end
+      # Changing the mode of sriov interface to switchdev mode
+      %x{devlink dev eswitch set pci/#{get_interface_pci} mode switchdev}
+      %x{ethtool -K #{sriov_get_interface} hw-tc-offload on}
+      if cur_value == "0x15b3"
+        # Binding virtual functions
+        vfs_pcis.each do|vfs_pci|
+          File.write("/sys/bus/pci/drivers/mlx5_core/bind",vfs_pci)
+        end
+      end
+    end
+  end
+
   def sriov_numvfs_path
     "/sys/class/net/#{sriov_get_interface}/device/sriov_numvfs"
   end
 
   def sriov_get_interface
-    resource[:name].split(':', 2).first
+    resource[:name].split(':', 3).first
   end
 
   def sriov_numvfs_value
-    resource[:name].split(':', 2).last.to_i
+    resource[:name].split(':', 3)[1].to_i
+  end
+
+  def vendor_path
+    "/sys/class/net/#{sriov_get_interface}/device/vendor"
+  end
+
+  def ovs_mode
+    if resource[:name].split(':', 3).length == 2
+      'legacy'
+    else
+      resource[:name].split(':', 3).last
+    end
+  end
+
+  def get_vfs_pcis
+    %x{cat /sys/class/net/#{sriov_get_interface}/device/virtfn*/uevent | grep PCI_SLOT_NAME | cut -d'=' -f2}.split(/\n+/)
+  end
+
+  def get_interface_pci
+    %x{ethtool -i #{sriov_get_interface} | grep bus-info | awk {'print$2'}}.strip
   end
 
 end

lib/puppet/type/sriov_vf_config.rb

@@ -3,8 +3,8 @@ Puppet::Type.newtype(:sriov_vf_config) do
   ensurable
 
   newparam(:name) do
-    desc "sriov_numvfs conf as <physical_network>:<number_of_vfs> format"
-    newvalues(/^[a-z0-9\-_]+:[0-9]+$/)
+    desc "sriov_numvfs conf as <physical_network>:<number_of_vfs>:<mode> format"
+    newvalues(/^[a-z0-9\-_]+:[0-9]+(:(switchdev|legacy))?$/)
   end
 
 end

manifests/host/sriov.pp

@@ -5,10 +5,13 @@
 # === Parameters
 #
 # [*number_of_vfs*]
-#   (optional) List of <physical_network>:<number_of_vfs> specifying the number
-#   VFs to be exposed per physical interface.
+#   (optional) List of <physical_network>:<number_of_vfs>:<sriov_mode>
+#   specifying the number VFs to be exposed per physical interface with sriov
+#   mode, where <sriov_mode> is optional field which accepts legacy or
+#   switchdev, and if it's not specified we default it to legacy.
 #   For example, to configure two interface with number of VFs, specify
-#   it as ['eth1:4','eth2:10']
+#   it as ['eth1:4','eth2:10:legacy'] for legacey mode or specify it as
+#   ['eth1:4:switchdev'] for switchdev mode
 #   Defaults to []
 #
 class tripleo::host::sriov (

manifests/host/sriov/numvfs_persistence.pp

@@ -62,7 +62,20 @@ define tripleo::host::sriov::numvfs_persistence(
     $vfspec = split($vf_defs[0], ':')
     $interface = $vfspec[0]
     $count = $vfspec[1]
-    $vfdef_str = "${content_string}[ \"${interface}\" == \"\$1\" ] && echo ${count} > /sys/class/net/${interface}/device/sriov_numvfs\n"
+    if (length($vfspec) == 3) {
+      $mode = $vfspec[2]
+    } else {
+      $mode = 'legacy'
+    }
+    if ($mode == 'switchdev') {
+      $vfdef_str = epp('tripleo/switchdev/switchdev.epp', {
+        'content_string' => "${content_string}",
+        'interface' => "${interface}",
+        'count' => "${count}"
+        })
+    } else {
+      $vfdef_str = "${content_string}[ \"${interface}\" == \"\$1\" ] && echo ${count} > /sys/class/net/${interface}/device/sriov_numvfs\n"
+    }
     $udev_str = "${udev_rules}KERNEL==\"${interface}\", RUN+=\"/etc/sysconfig/allocate_vfs %k\"\n"
     tripleo::host::sriov::numvfs_persistence{"mapped ${interface}":
       vf_defs        => delete_at($vf_defs, 0),

releasenotes/notes/ovs-hw-offload-89a49899af3b9892.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Allows to configure SR-IOV NIC to switchdev mode.
+    This feature requires kernel 4.10 and above.

spec/defines/tripleo_host_sriov_numvfs_persistence_spec.rb

@@ -47,3 +47,71 @@ describe 'tripleo::host::sriov::numvfs_persistence' do
     end
   end
 end
+
+describe 'tripleo::host::sriov::numvfs_persistence' do
+
+  describe 'confugure numvfs for persistence' do
+
+    let :title do
+      'numvfs'
+    end
+
+    let :params do
+      {
+        :name           => 'persistence',
+        :vf_defs        => ['eth0:10:switchdev','eth1:8:legacy'],
+        :content_string => "Hashbang\n",
+        :udev_rules     => ""
+      }
+    end
+
+    it 'configures persistence' do
+      is_expected.to contain_file('/etc/sysconfig/allocate_vfs').with(
+        :ensure  => 'file',
+        :content => "Hashbang\nif [ \"eth0\" == \"$1\" ]
+then
+  echo 10 > /sys/class/net/eth0/device/sriov_numvfs
+  if [ `cat /sys/class/net/eth0/device/vendor` == \"0x15b3\" ]
+  then
+    for pci in `cat /sys/class/net/eth0/device/virtfn*/uevent | grep PCI_SLOT_NAME | cut -d'=' -f2`
+    do
+      echo \$pci > /sys/bus/pci/drivers/mlx5_core/unbind
+    done
+  fi
+  interface_pci=`ethtool -i eth0 | grep bus-info | awk {'print\$2'}`
+  devlink dev eswitch set pci/\$interface_pci mode switchdev
+  ethtool -K eth0 hw-tc-offload on
+  if [ `cat /sys/class/net/eth0/device/vendor` == \"0x15b3\" ]
+  then
+    for pci in `cat /sys/class/net/eth0/device/virtfn*/uevent | grep PCI_SLOT_NAME | cut -d'=' -f2`
+    do
+      echo \$pci > /sys/bus/pci/drivers/mlx5_core/bind;
+    done
+  fi
+fi\n[ \"eth1\" == \"\$1\" ] && echo 8 > /sys/class/net/eth1/device/sriov_numvfs\n",
+        :group   => 'root',
+        :mode    => '0755',
+        :owner   => 'root',
+      )
+      is_expected.to contain_file('/sbin/ifup-local').with(
+        :group   => 'root',
+        :mode    => '0755',
+        :owner   => 'root',
+        :content => '#!/bin/bash',
+        :replace => false,
+      )
+      is_expected.to contain_file('/etc/udev/rules.d/70-tripleo-reset-sriov.rules').with(
+        :ensure  => 'file',
+        :content => "KERNEL==\"eth0\", RUN+=\"/etc/sysconfig/allocate_vfs %k\"\nKERNEL==\"eth1\", RUN+=\"/etc/sysconfig/allocate_vfs %k\"\n",
+        :group   => 'root',
+        :mode    => '0755',
+        :owner   => 'root',
+        :replace => true
+      )
+      is_expected.to contain_file_line('call_ifup-local').with(
+        :path => '/sbin/ifup-local',
+        :line => '/etc/sysconfig/allocate_vfs $1',
+      )
+    end
+  end
+end

spec/unit/provider/sriov_vf_config/numvfs_spec.rb

@@ -35,6 +35,83 @@ describe provider_class do
     it 'should return path of the file to enable vfs' do
       expect(provider.sriov_numvfs_path).to eql('/sys/class/net/eth0/device/sriov_numvfs')
     end
+    it 'should return ovs mode: legacy' do
+      expect(provider.ovs_mode).to eql('legacy')
+    end
+  end
+
+  let :numvfs_conf_switchdev do
+    {
+      :name   => 'eth1:12:switchdev',
+      :ensure => 'present',
+    }
+  end
+
+  describe 'when setting the attributes' do
+    let :resource_switchdev do
+      Puppet::Type::Sriov_vf_config.new(numvfs_conf_switchdev)
+    end
+
+    let :provider_switchdev do
+      provider_class.new(resource_switchdev)
+    end
+
+    it 'should return the correct interface name' do
+      expect(provider_switchdev.sriov_get_interface).to eql('eth1')
+    end
+
+    it 'should return the correct numvfs value' do
+      expect(provider_switchdev.sriov_numvfs_value).to eql(12)
+    end
+
+    it 'should return path of the file to enable vfs' do
+      expect(provider_switchdev.sriov_numvfs_path).to eql('/sys/class/net/eth1/device/sriov_numvfs')
+    end
+
+    it 'should return path of the vendor file' do
+      expect(provider_switchdev.vendor_path).to eql('/sys/class/net/eth1/device/vendor')
+    end
+
+    it 'should return ovs mode: switchdev' do
+      expect(provider_switchdev.ovs_mode).to eql('switchdev')
+    end
+  end
+
+  let :numvfs_conf_legacy do
+    {
+      :name   => 'eth2:14:legacy',
+      :ensure => 'present',
+    }
+  end
+
+  describe 'when setting the attributes' do
+    let :resource_legacy do
+      Puppet::Type::Sriov_vf_config.new(numvfs_conf_legacy)
+    end
+
+    let :provider_legacy do
+      provider_class.new(resource_legacy)
+    end
+
+    it 'should return the correct interface name' do
+      expect(provider_legacy.sriov_get_interface).to eql('eth2')
+    end
+
+    it 'should return the correct numvfs value' do
+      expect(provider_legacy.sriov_numvfs_value).to eql(14)
+    end
+
+    it 'should return path of the file to enable vfs' do
+      expect(provider_legacy.sriov_numvfs_path).to eql('/sys/class/net/eth2/device/sriov_numvfs')
+    end
+
+    it 'should return path of the vendor file' do
+      expect(provider_legacy.vendor_path).to eql('/sys/class/net/eth2/device/vendor')
+    end
+
+    it 'should return ovs mode: legacy' do
+      expect(provider_legacy.ovs_mode).to eql('legacy')
+    end
   end
 
 end

spec/unit/type/sriov_vf_config_spec.rb

@@ -44,4 +44,29 @@ describe 'Puppet::Type.type(:sriov_vf_config)' do
       :ensure => 'present'
     )}.to raise_error(Puppet::ResourceError)
   end
+  it 'should allow name to be passed' do
+    expect{Puppet::Type.type(:sriov_vf_config).new(
+      :name   => 'eth0:10:legacy',
+      :ensure => 'present'
+    )}.not_to raise_error
+  end
+  it 'should allow name to be passed' do
+    expect{Puppet::Type.type(:sriov_vf_config).new(
+      :name   => 'eth0:10:switchdev',
+      :ensure => 'present'
+    )}.not_to raise_error
+  end
+  it 'should throw error for invalid format for ovs mode' do
+    expect{Puppet::Type.type(:sriov_vf_config).new(
+      :name   => 'eth0:10:None',
+      :ensure => 'present'
+    )}.to raise_error(Puppet::ResourceError)
+  end
+  it 'should throw error for invalid format without ovs mode' do
+    expect{Puppet::Type.type(:sriov_vf_config).new(
+      :name   => 'eth0:10:',
+      :ensure => 'present'
+    )}.to raise_error(Puppet::ResourceError)
+  end
+
 end

templates/switchdev/switchdev.epp

@@ -0,0 +1,22 @@
+<%- | String $content_string = '', String $interface = '', String $count = '' | -%>
+<%=$content_string%>if [ "<%=$interface%>" == "$1" ]
+then
+  echo <%=$count%> > /sys/class/net/<%=$interface%>/device/sriov_numvfs
+  if [ `cat /sys/class/net/<%=$interface%>/device/vendor` == "0x15b3" ]
+  then
+    for pci in `cat /sys/class/net/<%=$interface%>/device/virtfn*/uevent | grep PCI_SLOT_NAME | cut -d'=' -f2`
+    do
+      echo $pci > /sys/bus/pci/drivers/mlx5_core/unbind
+    done
+  fi
+  interface_pci=`ethtool -i <%=$interface%> | grep bus-info | awk {'print$2'}`
+  devlink dev eswitch set pci/$interface_pci mode switchdev
+  ethtool -K <%=$interface%> hw-tc-offload on
+  if [ `cat /sys/class/net/<%=$interface%>/device/vendor` == "0x15b3" ]
+  then
+    for pci in `cat /sys/class/net/<%=$interface%>/device/virtfn*/uevent | grep PCI_SLOT_NAME | cut -d'=' -f2`
+    do
+      echo $pci > /sys/bus/pci/drivers/mlx5_core/bind;
+    done
+  fi
+fi