x/fuel-plugin-contrail
Code Issues Proposed changes

Bonding support for DPDK on VF

Browse Source 
- get_sriov_devices function updated
- VF configuration moved to a defined type
- get_vf_data function updated
- generate_passthrough_whitelist fuction updated

Change-Id: Idbcdd5e7a574d2a8474132ece43aef1776eccbe8
Signed-off-by: Illia Polliul <ipolliul@mirantis.com>
This commit is contained in:
Oleksandr Martsyniuk 2016-09-19 20:37:05 +03:00 committed by Illia Polliul
parent 66ae6b04ed
commit 2d7151fea9
10 changed files with 314 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
deployment_scripts/puppet/manifests/contrail-compute-dpdkonvf.pp
View File

@ -18,6 +18,6 @@ include contrail
# Create virtual functions for DPDK VF
if $contrail::compute_dpdk_on_vf {
$sriov_hash = get_sriov_devices($contrail::compute_dpdk_on_vf, $contrail::phys_dev)
$sriov_hash = get_sriov_devices($contrail::phys_dev)
create_resources(contrail::create_vfs, $sriov_hash)
}

46
deployment_scripts/puppet/modules/contrail/lib/puppet/parser/functions/generate_passthrough_whitelist.rb
View File

@ -22,31 +22,45 @@ module Puppet::Parser::Functions
physnet = args[0]
dpdk_on_vf = args[1]
sriov_hash = function_get_sriov_devices([])
#sriov_hash = function_get_sriov_devices([])
network_scheme = function_hiera_hash(['network_scheme', {}])
interfaces = Array.new()
list = []
if function_get_nic_passthrough_whitelist(['sriov'])
list += function_get_nic_passthrough_whitelist(['sriov'])
end
if dpdk_on_vf
hiera_data_key = "priv_int_vfn_wl"
priv_int = args[2].sub(/\..*/, '')
interface = args[2].split('.').first
dpdk_vf_number = args[3]
if (File.exists?("/sys/class/net/#{priv_int}"))
vfn = Dir.glob "/sys/class/net/#{priv_int}/device/virtfn*"
vfn_wl = vfn.map { |f|
if not f.end_with? "virtfn#{dpdk_vf_number}"
pci_address = File.readlink(f).split("/")[1]
Hash["address" => pci_address, "physical_network" => physnet]
end
}
list += vfn_wl
function_add_data_to_yaml(["/etc/hiera/plugins/contrail.yaml", hiera_data_key, vfn_wl])
elsif not function_hiera_array([hiera_data_key, []]).empty?
vfn_wl = function_hiera_array([hiera_data_key, []])
list += vfn_wl
network_scheme = function_hiera_hash(['network_scheme', {}])
network_scheme['transformations'].each do |entry|
if entry['name'] == interface and entry.has_key?('bond_properties') and entry['action'] == "add-bond"
interfaces.push(*entry['interfaces'])
elsif entry['action'] == 'add-port' and entry['name'] == args[0]
interfaces << interface
end
end
interfaces.each do |interface|
if (File.exists?("/sys/class/net/#{interface}"))
vfn = Dir.glob "/sys/class/net/#{interface}/device/virtfn*"
vfn_wl = vfn.map { |f|
if not f.end_with? "virtfn#{dpdk_vf_number}"
pci_address = File.readlink(f).split("/")[1]
Hash["address" => pci_address, "physical_network" => physnet]
end
}
list += vfn_wl.compact
function_add_data_to_yaml(["/etc/hiera/plugins/contrail.yaml", hiera_data_key, vfn_wl.compact])
elsif not function_hiera_array([hiera_data_key, []]).empty?
vfn_wl = function_hiera_array([hiera_data_key, []])
list = vfn_wl
end
end
end

46
deployment_scripts/puppet/modules/contrail/lib/puppet/parser/functions/get_bond_name.rb Normal file
View File

@ -0,0 +1,46 @@
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
module L23network
module Puppet::Parser::Functions
newfunction(:get_bond_name, :type => :rvalue, :doc => <<-EOS
Returns bond name, if overrides are not applied
1) Get network_scheme including overrides
2) Check if bond is present in scheme
3) If yes, return raw bond name, otherwise return undef
example:
get_bond_name($network_scheme, $contrail::phys_dev, $contrail::compute_dpdk_on_vf)
used in dpdk-on-vf to determine if bond needs to be created
EOS
) do |argv|
cfg_hash, interface, dpdk_enabled = argv
interface_real = interface.split('.').first
rv = nil
# this part is a copy from prepare_network_config.rb
ns = L23network.sanitize_bool_in_hash(L23network.sanitize_keys_in_hash(cfg_hash))
ns = L23network.override_transformations(ns)
ns = L23network.remove_empty_members(ns)
ns[:transformations].each do |t|
if t[:action] == 'add-bond' and t[:name] == interface_real and dpdk_enabled
rv = interface_real
end
end
return rv
end
end
end

47
deployment_scripts/puppet/modules/contrail/lib/puppet/parser/functions/get_sriov_devices.rb
View File

@ -16,40 +16,35 @@ module Puppet::Parser::Functions
newfunction(:get_sriov_devices, :type => :rvalue, :doc => <<-EOS
Returns sriov capable devices
example:
get_sriov_devices()
get_sriov_devices('phys_dev')
EOS
) do |args|
dpdk_on_vf = args[0]
bridge_interfaces = Array.new()
bond_interfaces = Array.new()
interface = args[0].split('.').first
interfaces = Array.new()
sriov_hash = Hash.new
network_scheme = function_hiera_hash(['network_scheme', {}])
network_scheme['transformations'].each do |entry|
if entry.has_key?('bridge') and entry['action'] == "add-port"
bridge_interfaces.push(entry['name'])
end
if entry.has_key?('bond_properties') and entry['action'] == "add-bond"
bond_interfaces.push(*entry['interfaces'])
end
if entry['name'] == interface and entry.has_key?('bond_properties') and entry['action'] == "add-bond"
interfaces.push(*entry['interfaces'])
elsif entry['action'] == 'add-port' and entry['name'] == args[0]
interfaces << interface
end
end
sriov_hash = Hash.new
if dpdk_on_vf
hiera_data_key = "priv_int_sriov_data"
private_interface = args[1].sub(/\..*/, '')
private_interface_path = "/sys/class/net/" + private_interface
if (File.exists?(private_interface_path + "/device/sriov_totalvfs"))
sriov_hash[private_interface] = Hash.new
sriov_hash[private_interface]["totalvfs"] = IO.read(private_interface_path + "/device/sriov_totalvfs").to_i
sriov_hash[private_interface]["numvfs"] = IO.read(private_interface_path + "/device/sriov_numvfs").to_i
function_add_data_to_yaml(["/etc/hiera/plugins/contrail.yaml", hiera_data_key, sriov_hash[private_interface]])
elsif not function_hiera_hash([hiera_data_key, {}]).empty?
sriov_hiera = function_hiera_hash([hiera_data_key, {}])
sriov_hash[private_interface] = Hash.new
sriov_hash[private_interface]["totalvfs"] = sriov_hiera["totalvfs"]
sriov_hash[private_interface]["numvfs"] = sriov_hiera["numvfs"]
interfaces.each do |interface|
interface_path = "/sys/class/net/" + interface
if (File.exists?(interface_path + "/device/sriov_totalvfs"))
sriov_hash[interface] = Hash.new
sriov_hash[interface]["totalvfs"] = IO.read(interface_path + "/device/sriov_totalvfs").to_i - 1
sriov_hash[interface]["numvfs"] = IO.read(interface_path + "/device/sriov_numvfs").to_i
function_add_data_to_yaml(["/etc/hiera/plugins/contrail.yaml", "sriov_hash", sriov_hash])
elsif function_hiera_hash(['sriov_hash', {}]).has_key?(interface)
sriov_hiera = function_hiera_hash(['sriov_hash', {}])
sriov_hash[interface] = Hash.new
sriov_hash[interface]["totalvfs"] = sriov_hiera["totalvfs"]
sriov_hash[interface]["numvfs"] = sriov_hiera["numvfs"]
end
end

20
deployment_scripts/puppet/modules/contrail/lib/puppet/parser/functions/get_vf_data.rb
View File

@ -16,24 +16,28 @@ module Puppet::Parser::Functions
newfunction(:get_vf_data, :type => :rvalue, :doc => <<-EOS
Returns interface name, pci address, mac address related to specific virtual function
example:
get_vf_data(dev_name, vf_number)
get_vf_data(pf_dev_name, vf_number)
EOS
) do |args|
dev_name = args[0].sub(/\..*/, '')
pf_dev_name = args[0].split('.').first
vf_number = args[1]
vf_sys = "/sys/class/net/#{dev_name}/device/virtfn#{vf_number}"
vf_sys = "/sys/class/net/#{pf_dev_name}/device/virtfn#{vf_number}"
vf_data = Hash.new
hiera_data_key = "dpdk_vf_int_data"
vf_prefix = lookupvar('contrail::vf_prefix')
if (File.exists?("/sys/class/net/#{dev_name}/device/virtfn#{vf_number}/net"))
# this is what VF name would be after renaming
vf_name = vf_prefix + pf_dev_name
if (File.exists?("/sys/class/net/#{pf_dev_name}/device/virtfn#{vf_number}/net"))
# this is original VF name, before renaming
vf_dev_name = Dir.entries("#{vf_sys}/net/")[2]
vf_pci_addr = File.readlink(vf_sys).split("/")[1]
vf_mac_addr = File.open("#{vf_sys}/net/#{vf_dev_name}/address", "rb") { |f| f.read.strip }
vf_data = {"vf_dev_name" => vf_dev_name, "vf_pci_addr" => vf_pci_addr, "vf_mac_addr" => vf_mac_addr}
function_add_data_to_yaml(["/etc/hiera/plugins/contrail.yaml", hiera_data_key, vf_data])
elsif not function_hiera_hash([hiera_data_key, {}]).empty?
vf_data = function_hiera_hash([hiera_data_key, {}])
function_add_data_to_yaml(["/etc/hiera/plugins/contrail.yaml", vf_name, vf_data])
elsif not function_hiera_hash([vf_name, {}]).empty?
vf_data = function_hiera_hash([vf_name, {}])
end
return vf_data

103
deployment_scripts/puppet/modules/contrail/manifests/compute/network.pp
View File

@ -21,6 +21,7 @@ class contrail::compute::network {
Exec {
provider => 'shell',
path => '/usr/bin:/bin:/sbin:/usr/sbin',
logoutput => true,
}
file { '/etc/network/interfaces.d/ifcfg-br-mesh':
@ -36,45 +37,82 @@ class contrail::compute::network {
returns => [0,1] # Idempotent
}
define ifup {
exec { "ifup_${name}":
command => "sleep 20; ifup ${name}",
}
}
# Configure persistent network device for DPDK VF
if $contrail::compute_dpdk_on_vf {
$vlan_tag = regsubst($::contrail::phys_dev, '^.+\.' , '')
$vf_data = get_vf_data($contrail::phys_dev, $contrail::dpdk_vf_number)
$dpdk_dev_name = "dpdk-vf${contrail::dpdk_vf_number}"
$dpdk_vf_origin_name = $vf_data['vf_dev_name']
$dpdk_dev_pci = $vf_data['vf_pci_addr']
$dpdk_dev_mac = $vf_data['vf_mac_addr']
$raw_phys_dev = regsubst($::contrail::phys_dev, '\..*' , '')
exec { 'rename-dpdk-vf':
command => "ip link set ${dpdk_vf_origin_name} name ${dpdk_dev_name}",
unless => "ip link | grep ${dpdk_dev_name}",
}
$phys_dev = $contrail::phys_dev
$vlan_tag = regsubst($phys_dev, '^.+\.' , '')
$raw_phys_dev = regsubst($phys_dev, '\..*' , '')
# for file_line resources in configure_vfs we need file to be precreated
file {'/etc/udev/rules.d/72-contrail-dpdk-on-vf.rules':
ensure => present,
content => template('contrail/72-contrail-dpdk-on-vf.rules.erb'),
}
$interface_config = join(["auto ${dpdk_dev_name}",
"iface ${dpdk_dev_name} inet manual",
"pre-up ip link set link dev ${raw_phys_dev} vf ${contrail::dpdk_vf_number} vlan 0",
"post-up ip link set link dev ${raw_phys_dev} vf ${contrail::dpdk_vf_number} spoof off",
],"\n")
$sriov_hash = get_sriov_devices($contrail::phys_dev)
file {"/etc/network/interfaces.d/ifcfg-${dpdk_dev_name}":
ensure => file,
content => $interface_config,
require => File['/etc/udev/rules.d/72-contrail-dpdk-on-vf.rules'],
}
# this funcition will return undef if overrides are in place
# otherwise returns bond raw name
$bond = get_bond_name($contrail::network_scheme, $contrail::phys_dev, $contrail::compute_dpdk_on_vf)
if $bond {
include l23network
exec {"destroy_old_${raw_phys_dev}":
command => "ifdown ${raw_phys_dev}",
unless => "grep dpdk_vf /etc/network/interfaces.d/ifcfg-${raw_phys_dev}",
}
exec { "ifup_${dpdk_dev_name}":
command => "ifup ${dpdk_dev_name}",
unless => "ip link show dev ${dpdk_dev_name} | grep ,UP",
require => File["/etc/network/interfaces.d/ifcfg-${dpdk_dev_name}"],
$sriov_defaults = {
'bond_dev_name' => $bond,
}
create_resources(contrail::configure_vfs, $sriov_hash, $sriov_defaults)
$bond_vfs = keys(prefix($sriov_hash, $contrail::vf_prefix))
$slaves = join($bond_vfs, ' ')
l23network::l2::bond { 'bond0':
interfaces => [$bond_vfs],
mtu => 1500,
onboot => true,
bond_properties => { # bond configuration properties (see bonding.txt)
mode => 'balance-rr',
},
delay_while_up => 15,
provider => lnx,
}
file {'/etc/modprobe.d/ixgbe.conf':
ensure => file,
content => 'options ixgbe max_vfs=10',
} ->
exec { 'update-initramfs':
command => 'update-initramfs -u',
}
Exec['flush_addr_br_mesh'] -> Exec["destroy_old_${raw_phys_dev}"] ->
Configure_vfs <||> -> L23Network::L2::Bond <||> ->
ifup{ $bond_vfs: }
} else {
create_resources(contrail::configure_vfs, $sriov_hash)
}
# Add vlan interface config if needed
if ($::contrail::phys_dev != $raw_phys_dev) {
# In case of bond, this is skipped,
# vlan interface config provided by l23network
if ($vlan_tag =~ /^\d*$/ and count(keys($sriov_hash)) == 1 ) {
$sriov_ifaces = keys($sriov_hash)
$pf_dev_name = $sriov_ifaces[0]
$dpdk_dev_name = "${contrail::vf_prefix}${pf_dev_name}"
$vlan_interface_config = join([ "auto ${dpdk_dev_name}.${vlan_tag}",
"iface ${dpdk_dev_name}.${vlan_tag} inet manual",
@ -84,21 +122,16 @@ class contrail::compute::network {
file {"/etc/network/interfaces.d/ifcfg-${dpdk_dev_name}.${vlan_tag}":
ensure => file,
content => $vlan_interface_config,
require => [File['/etc/udev/rules.d/72-contrail-dpdk-on-vf.rules'],
File["/etc/network/interfaces.d/ifcfg-${dpdk_dev_name}"],
],
}
Configure_vfs <||> ->
exec { "ifup_${dpdk_dev_name}.${vlan_tag}":
command => "ifup ${dpdk_dev_name}.${vlan_tag}",
unless => "ip link show dev ${dpdk_dev_name}.${vlan_tag} | grep ,UP",
require => [File["/etc/network/interfaces.d/ifcfg-${dpdk_dev_name}.${vlan_tag}"],
Exec["ifup_${dpdk_dev_name}"],
],
File["/etc/network/interfaces.d/ifcfg-${dpdk_dev_name}"],
]
}
}
}
}

43
deployment_scripts/puppet/modules/contrail/manifests/compute/vrouter.pp
View File

@ -18,22 +18,41 @@ class contrail::compute::vrouter {
$vlan_tag = regsubst($::contrail::phys_dev, '^.+\.' , '')
if $contrail::compute_dpdk_on_vf {
$vf_data = get_vf_data($contrail::phys_dev, $contrail::dpdk_vf_number)
# Add vlan to phys_dev if needed
if ($::contrail::phys_dev != $raw_phys_dev) {
$phys_dev = "dpdk-vf${contrail::dpdk_vf_number}.${vlan_tag}"
$sriov_hash = get_sriov_devices($contrail::phys_dev)
$sriov_ifaces = keys($sriov_hash)
$pf_dev_name = $sriov_ifaces[0]
$dpdk_dev_name = "${contrail::vf_prefix}${pf_dev_name}"
# unbonded VFs go here
if count(keys($sriov_hash)) == 1 {
if $vlan_tag =~ /^\d*$/ {
$phys_dev = "${dpdk_dev_name}.${vlan_tag}"
} else {
$phys_dev = $dpdk_dev_name
}
$vf_data = get_vf_data($pf_dev_name, $contrail::dpdk_vf_number)
$dpdk_dev_pci = $vf_data['vf_pci_addr']
$dev_mac = $vf_data['vf_mac_addr']
# bonds from VFs go here
} else {
$phys_dev = "dpdk-vf${contrail::dpdk_vf_number}"
$phys_dev = $contrail::phys_dev
$dev_mac = getvar("::macaddress_${dpdk_dev_name}")
$dpdk_dev_pci = $contrail::phys_dev_pci
}
$dpdk_dev_pci = $vf_data['vf_pci_addr']
$dev_mac = $vf_data['vf_mac_addr']
# usual dpdk and kernel-based routers go here
} else {
# facter uses underscore instead of dot as a separator for interface name with vlan
$phys_dev_facter = regsubst($::contrail::phys_dev, '\.' , '_')
$dev_mac = getvar("::macaddress_${phys_dev_facter}")
$phys_dev = $contrail::phys_dev
$dpdk_dev_pci = $contrail::phys_dev_pci
# facter uses underscore instead of dot as a separator for interface name with vlan
$phys_dev_facter = regsubst($::contrail::phys_dev, '\.' , '_')
$dev_mac = getvar("::macaddress_${phys_dev_facter}")
$phys_dev = $contrail::phys_dev
$dpdk_dev_pci = $contrail::phys_dev_pci
}
Exec {

89
deployment_scripts/puppet/modules/contrail/manifests/configure_vfs.pp Normal file
View File

@ -0,0 +1,89 @@
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
define contrail::configure_vfs (
$numvfs,
$totalvfs,
$pf_dev_name = $title,
$bond_dev_name = undef,
) {
Exec {
provider => 'shell',
path => '/usr/bin:/bin:/sbin:/usr/sbin',
}
$vf_data = get_vf_data($pf_dev_name, $contrail::dpdk_vf_number)
$vf_dev_name = "${contrail::vf_prefix}${pf_dev_name}"
$vf_number = $contrail::dpdk_vf_number
$vf_origin_name = $vf_data['vf_dev_name']
$vf_dev_pci = $vf_data['vf_pci_addr']
$vf_dev_mac = $vf_data['vf_mac_addr']
$udev_rule = join(['SUBSYSTEM=="net"',
'ACTION=="add"',
"KERNELS==\"${vf_dev_pci}\"",
"NAME=\"${vf_dev_name}\"",
"RUN+=\"/bin/ip link set dev %k address ${vf_dev_mac}\"",
],', ')
$interface_config_array = ["auto ${vf_dev_name}",
"iface ${vf_dev_name} inet manual",
"hwaddress ether ${vf_dev_mac}",
"pre-up ip link set link dev ${pf_dev_name} vf ${vf_number} spoof off"
]
$dpdk_vrouter_up = 'pgrep -f "^/usr/bin/contrail-vrouter-dpdk"'
# $dpdk_vrouter_up = 'false'
if !$bond_dev_name {
$interface_config=join(concat($interface_config_array, "pre-up ip link set link dev ${pf_dev_name} vf ${vf_number} vlan 0"), "\n")
file {"/etc/network/interfaces.d/ifcfg-${vf_dev_name}":
ensure => file,
content => $interface_config,
}
}
file_line {"udev_rule_for_${vf_dev_name}":
line => $udev_rule,
path => '/etc/udev/rules.d/72-contrail-dpdk-on-vf.rules',
require => File['/etc/udev/rules.d/72-contrail-dpdk-on-vf.rules'],
} ->
exec { "ifup_${pf_dev_name}":
command => "ifup ${pf_dev_name}",
unless => ["ip link show ${pf_dev_name} | grep ,UP",
$dpdk_vrouter_up]
} ->
exec { "rename-${vf_dev_name}":
command => "ip link set ${vf_origin_name} name ${vf_dev_name}",
unless => ["ip link | grep ${vf_dev_name}",
$dpdk_vrouter_up]
}
if !$bond_dev_name {
exec { "ifup_${vf_dev_name}":
command => "ifup ${vf_dev_name}",
unless => ["ip link show ${vf_dev_name} | grep ,UP",
$dpdk_vrouter_up],
require => [File["/etc/network/interfaces.d/ifcfg-${vf_dev_name}"],
Exec["rename-${vf_dev_name}"]]
}
}
}

23
deployment_scripts/puppet/modules/contrail/manifests/create_vfs.pp
View File

@ -25,12 +25,20 @@ define contrail::create_vfs (
path => '/usr/bin:/bin:/sbin',
}
$final_vf = min(63, $totalvfs)
$bond = get_bond_name($contrail::network_scheme, $contrail::phys_dev, $contrail::compute_dpdk_on_vf)
$interface_config = join(["auto ${network_device}",
"iface ${network_device} inet manual",
"post-up echo ${final_vf} > /sys/class/net/${network_device}/device/sriov_numvfs"
],"\n")
$interface_config_array = [ "auto ${network_device}",
"iface ${network_device} inet manual",
"post-up echo ${totalvfs} > /sys/class/net/${network_device}/device/sriov_numvfs",
"post-up ip link set link dev ${network_device} vf ${contrail::dpdk_vf_number} spoof off", '',
]
if !$bond {
$interface_config=join(concat($interface_config_array, "post-up ip link set link dev ${network_device} vf ${contrail::dpdk_vf_number} vlan 0"), "\n")
} else {
$interface_config=join($interface_config_array, "\n")
}
file {"/etc/network/interfaces.d/ifcfg-${network_device}":
ensure => file,
@ -38,8 +46,7 @@ define contrail::create_vfs (
}
exec {"create_vfs_on_${network_device}":
command => "echo ${final_vf} > /sys/class/net/${network_device}/device/sriov_numvfs",
tag => ['create_vfs'],
command => "echo ${totalvfs} > /sys/class/net/${network_device}/device/sriov_numvfs",
}
}
}

2
deployment_scripts/puppet/modules/contrail/manifests/init.pp
View File

@ -116,7 +116,7 @@ class contrail {
$compute_dpdk_on_vf = $compute_dpdk_enabled and roles_include('dpdk-on-vf')
$dpdk_physnet = $settings['dpdk_physnet']
$dpdk_vf_number = 0
$vf_prefix = 'vf_'
# ToR/TSN agent settings
$enable_tor_agents = $settings['enable_tor_agents']
if $enable_tor_agents == true {