DPDK on VF feature updated to support core SR-IOV

- generate_passthrough_whitelist function updated
- create_vfs puppet manifest added
- dpdk_on_vf manifest updated
- vrouter manifest updated
- reboot task removed
- typos fixed

Change-Id: I4ddf1676d39844567390f29567abbd83478d7f76
Signed-off-by: Illia Polliul <ipolliul@mirantis.com>
This commit is contained in:
Oleksandr Martsyniuk 2016-09-14 11:51:15 +03:00 committed by Illia Polliul
parent 59a207fbe7
commit d6ce5137c2
15 changed files with 216 additions and 210 deletions

View File

@ -12,7 +12,12 @@
# License for the specific language governing permissions and limitations
# under the License.
notice('MODULAR: contrail/contrail-compute-dpdk-on-vf.pp')
notice('MODULAR: contrail/contrail-compute-dpdkonvf.pp')
include contrail
class { 'contrail::compute::dpdk_on_vf': }
# Create virtual functions for DPDK VF
if $contrail::compute_dpdk_on_vf {
$sriov_hash = get_sriov_devices($contrail::compute_dpdk_on_vf, $contrail::phys_dev)
create_resources(contrail::create_vfs, $sriov_hash)
}

View File

@ -25,10 +25,7 @@ module Puppet::Parser::Functions
sriov_hash = function_get_sriov_devices([])
network_scheme = function_hiera_hash(['network_scheme', {}])
list = sriov_hash.map { |dev, _|
pci_address = function_get_dev_pci_addr([dev, network_scheme])
Hash["address" => pci_address, "physical_network" => physnet]
}
list = function_get_nic_passthrough_whitelist(['sriov'])
if dpdk_on_vf
hiera_data_key = "priv_int_vfn_wl"

View File

@ -36,22 +36,6 @@ module Puppet::Parser::Functions
sriov_hash = Hash.new
Dir.foreach('/sys/class/net') do |network_interface|
next if network_interface == '.' or network_interface == '..'
network_interface_path = "/sys/class/net/" + network_interface
if (File.exists?(network_interface_path + "/device/sriov_totalvfs") and
not bridge_interfaces.include?(network_interface) and
not bond_interfaces.include?(network_interface))
system "ip link set #{network_interface} up"
sleep 30 # wait to be sure that interface became up
if IO.read(network_interface_path + "/operstate").to_s.strip == "up"
sriov_hash[network_interface] = Hash.new
sriov_hash[network_interface]["totalvfs"] = IO.read(network_interface_path + "/device/sriov_totalvfs").to_i
sriov_hash[network_interface]["numvfs"] = IO.read(network_interface_path + "/device/sriov_numvfs").to_i
end
end
end
if dpdk_on_vf
hiera_data_key = "priv_int_sriov_data"
private_interface = args[1]

View File

@ -13,7 +13,7 @@
# under the License.
module Puppet::Parser::Functions
newfunction(:get_fv_data, :type => :rvalue, :doc => <<-EOS
newfunction(:get_vf_data, :type => :rvalue, :doc => <<-EOS
Returns interface name, pci address, mac address related to specific virtual function
example:
get_vf_data(dev_name, vf_number)

View File

@ -1,63 +0,0 @@
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
class contrail::compute::dpdk_on_vf {
if $contrail::compute_dpkd_on_vf {
$vf_data = get_fv_data($contrail::phys_dev, $contrail::dpdk_vf_number)
$dpdk_dev_name = "dpdk-vf${contrail::dpdk_vf_number}"
$dpdk_vf_origin_name = $vf_data['vf_dev_name']
$dpdk_dev_pci = $vf_data['vf_pci_addr']
$dpdk_dev_mac = $vf_data['vf_mac_addr']
$phys_dev = $dpdk_dev_name
$pci_wl = generate_passthrough_whitelist(
$contrail::sriov_physnet,
$contrail::compute_dpkd_on_vf,
$contrail::phys_dev,
$contrail::dpdk_vf_number
)
exec { 'rename-dpdk-vf':
path => '/bin:/usr/bin:/usr/sbin',
command => "ip link set ${dpdk_vf_origin_name} name ${dpdk_dev_name}",
unless => 'ip link | grep vhost0',
}
file {'/etc/udev/rules.d/72-contrail-dpdk-on-vf.rules':
ensure => present,
content => template('contrail/72-contrail-dpdk-on-vf.rules.erb'),
}
file {'/etc/contrail/contrail-vrouter-agent.conf':
ensure => present,
content => template('contrail/contrail-vrouter-agent.conf.erb'),
}
nova_config {
'DEFAULT/pci_passthrough_whitelist': value => $pci_wl;
} ~>
service { 'nova-compute':
ensure => running,
enable => true,
}
service {'supervisor-vrouter':
ensure => running,
enable => true,
subscribe => [Exec['rename-dpdk-vf'],
File['/etc/contrail/contrail-vrouter-agent.conf']],
}
}
}

View File

@ -18,17 +18,14 @@ class contrail::compute::network {
$netmask = $contrail::netmask_short
$default_gw = undef
$br_file = $::operatingsystem ? {
'Ubuntu' => '/etc/network/interfaces.d/ifcfg-br-mesh',
'CentOS' => '/etc/sysconfig/network-scripts/ifcfg-br-mesh',
}
Exec {
provider => 'shell',
path => '/usr/bin:/bin:/sbin',
path => '/usr/bin:/bin:/sbin:/usr/sbin',
}
file { $br_file: ensure => absent } ->
file { '/etc/network/interfaces.d/ifcfg-br-mesh':
ensure => absent,
} ->
# Remove interface from the bridge
exec {"remove_${ifname}_mesh":
command => "brctl delif br-mesh ${ifname}",
@ -38,22 +35,46 @@ class contrail::compute::network {
command => 'ip addr flush dev br-mesh',
returns => [0,1] # Idempotent
}
case $::operatingsystem {
'Ubuntu': {
file {'/etc/network/interfaces.d/ifcfg-vhost0':
ensure => present,
content => template('contrail/ubuntu-ifcfg-vhost0.erb'),
}
# Configure persistent network device for DPDK VF
if $contrail::compute_dpdk_on_vf {
$vf_data = get_vf_data($contrail::phys_dev, $contrail::dpdk_vf_number)
$dpdk_dev_name = "dpdk-vf${contrail::dpdk_vf_number}"
$dpdk_vf_origin_name = $vf_data['vf_dev_name']
$dpdk_dev_pci = $vf_data['vf_pci_addr']
$dpdk_dev_mac = $vf_data['vf_mac_addr']
exec { 'rename-dpdk-vf':
command => "ip link set ${dpdk_vf_origin_name} name ${dpdk_dev_name}",
unless => "ip link | grep ${dpdk_dev_name}",
} ->
exec { 'set-dpdk-vf-vlan':
command => "ip link set link dev ${contrail::phys_dev} vf ${contrail::dpdk_vf_number} vlan 0"
}
'CentOS': {
exec {"remove_bridge_from_${ifname}_config":
command => "sed -i '/BRIDGE/d' /etc/sysconfig/network-scripts/ifcfg-${ifname}",
}
file {'/etc/sysconfig/network-scripts/ifcfg-vhost0':
ensure => present,
content => template('contrail/centos-ifcfg-vhost0.erb'),
}
exec { 'set-dpdk-vf-up':
command => "ip link set link dev ${dpdk_dev_name} up",
} ->
exec { 'set-dpdk-vf-spoof':
command => "ip link set link dev ${contrail::phys_dev} vf ${contrail::dpdk_vf_number} spoof off",
}
file {'/etc/udev/rules.d/72-contrail-dpdk-on-vf.rules':
ensure => present,
content => template('contrail/72-contrail-dpdk-on-vf.rules.erb'),
}
$interface_config = join(["auto ${dpdk_dev_name}",
"iface ${dpdk_dev_name} inet manual",
"pre-up ip link set link dev ${contrail::phys_dev} vf ${contrail::dpdk_vf_number} vlan 0",
"post-up ip link set link dev ${contrail::phys_dev} vf ${contrail::dpdk_vf_number} spoof off",
],"\n")
file {"/etc/network/interfaces.d/ifcfg-${dpdk_dev_name}":
ensure => file,
content => $interface_config,
require => File['/etc/udev/rules.d/72-contrail-dpdk-on-vf.rules'],
}
default: {}
}
}

View File

@ -35,7 +35,18 @@ class contrail::compute::nova {
'DEFAULT/heal_instance_info_cache_interval': value => '0';
}
if $contrail::compute_dpdk_enabled or $contrail::compute_sriov_enabled {
if $contrail::compute_dpdk_enabled {
if $contrail::compute_dpdk_on_vf {
$pci_wl = generate_passthrough_whitelist(
$contrail::dpdk_physnet,
$contrail::compute_dpdk_on_vf,
$contrail::phys_dev,
$contrail::dpdk_vf_number
)
nova_config {
'DEFAULT/pci_passthrough_whitelist': value => $pci_wl;
}
}
nova_config {
'libvirt/virt_type': value => 'kvm';
'CONTRAIL/use_userspace_vhost': value => true;

View File

@ -14,11 +14,18 @@
class contrail::compute::vrouter {
# facter uses underscore instead of dot as a separator for interface name with vlan
$phys_dev_facter = regsubst($::contrail::phys_dev, '\.' , '_')
$dev_mac = getvar("::macaddress_${phys_dev_facter}")
$phys_dev = $contrail::phys_dev
$dpdk_dev_pci = $contrail::phys_dev_pci
if $contrail::compute_dpdk_on_vf {
$vf_data = get_vf_data($contrail::phys_dev, $contrail::dpdk_vf_number)
$phys_dev = "dpdk-vf${contrail::dpdk_vf_number}"
$dpdk_dev_pci = $vf_data['vf_pci_addr']
$dev_mac = $vf_data['vf_mac_addr']
} else {
# facter uses underscore instead of dot as a separator for interface name with vlan
$phys_dev_facter = regsubst($::contrail::phys_dev, '\.' , '_')
$dev_mac = getvar("::macaddress_${phys_dev_facter}")
$phys_dev = $contrail::phys_dev
$dpdk_dev_pci = $contrail::phys_dev_pci
}
Exec {
path => '/sbin:/usr/sbin:/bin:/usr/bin',
@ -26,11 +33,15 @@ class contrail::compute::vrouter {
if $contrail::compute_dpdk_enabled {
$dpdk_mac = $::mac_from_vrouter
if $dpdk_mac {
$dpdk_dev_mac = $dpdk_mac
} else {
$mac_from_vrouter = $::mac_from_vrouter
if $contrail::compute_dpdk_on_vf {
$dpdk_dev_mac = $dev_mac
} else {
if $mac_from_vrouter {
$dpdk_dev_mac = $mac_from_vrouter
} else {
$dpdk_dev_mac = $dev_mac
}
}
$raw_phys_dev = regsubst($::contrail::phys_dev, '\..*' , '')
@ -114,72 +125,57 @@ class contrail::compute::vrouter {
require => Package[$install_packages],
}
if $contrail::compute_dpkd_on_vf {
$sriov_in_kernel = sriov_in_kernel()
$cmd_arr = ['puppet apply -v -d --logdest /var/log/puppet.log',
'--modulepath=/etc/puppet/modules/:/etc/fuel/plugins/contrail-4.0/puppet/modules/',
'/etc/fuel/plugins/contrail-4.0/puppet/manifests/contrail-compute-dpdk-on-vf.pp',
'&& sed -i "/contrail-compute-dpdk-on-vf/d" /etc/rc.local']
contrail_vrouter_agent_config {
'DEFAULT/log_file': value => '/var/log/contrail/contrail-vrouter-agent.log';
'DEFAULT/log_level': value => 'SYS_NOTICE';
'DEFAULT/log_local': value => '1';
'DEFAULT/log_flow': value => '1';
'DEFAULT/use_syslog': value => '1';
'DEFAULT/syslog_facility': value => 'LOG_LOCAL0';
'DEFAULT/headless_mode': value => $contrail::headless_mode;
'DISCOVERY/server': value => $contrail::contrail_private_vip;
'DISCOVERY/max_control_nodes': value => '2';
'HYPERVISOR/type': value => 'kvm';
'METADATA/metadata_proxy_secret': value => $contrail::metadata_secret;
'NETWORKS/control_network_ip': value => $contrail::address;
'VIRTUAL-HOST-INTERFACE/name': value => 'vhost0';
'VIRTUAL-HOST-INTERFACE/ip': value => "${contrail::address}/${contrail::netmask_short}";
'VIRTUAL-HOST-INTERFACE/physical_interface': value => $phys_dev;
'SERVICE-INSTANCE/netns_command': value => '/usr/bin/opencontrail-vrouter-netns';
} ->
if $sriov_in_kernel {
class { 'contrail::compute::dpdk_on_vf':
require => [Package[$install_packages],Exec['remove-ovs-modules'],
File['/etc/contrail/agent_param']],
}
} else {
file_line {'apply_dpdk_on_vf_after_reboot':
line => join($cmd_arr, ' '),
path => '/etc/rc.local',
}
ini_setting { 'vrouter-threadcount':
ensure => present,
path => '/etc/contrail/supervisord_vrouter.conf',
section => 'supervisord',
setting => 'environment',
value => 'TBB_THREAD_COUNT=8',
notify => Service['supervisor-vrouter'],
}
service {'supervisor-vrouter':
ensure => stopped,
enable => false,
require => Package[$install_packages],
}
}
if $contrail::gateway {
contrail_vrouter_agent_config { 'VIRTUAL-HOST-INTERFACE/gateway': value => $contrail::gateway; }
}
} else {
if $contrail::compute_dpdk_enabled == true {
contrail_vrouter_agent_config {
'DEFAULT/log_file': value => '/var/log/contrail/contrail-vrouter-agent.log';
'DEFAULT/log_level': value => 'SYS_NOTICE';
'DEFAULT/log_local': value => '1';
'DEFAULT/log_flow': value => '1';
'DEFAULT/use_syslog': value => '1';
'DEFAULT/syslog_facility': value => 'LOG_LOCAL0';
'DEFAULT/headless_mode': value => $contrail::headless_mode;
'DISCOVERY/server': value => $contrail::contrail_private_vip;
'DISCOVERY/max_control_nodes': value => '2';
'HYPERVISOR/type': value => 'kvm';
'METADATA/metadata_proxy_secret': value => $contrail::metadata_secret;
'NETWORKS/control_network_ip': value => $contrail::address;
'VIRTUAL-HOST-INTERFACE/name': value => 'vhost0';
'VIRTUAL-HOST-INTERFACE/ip': value => "${contrail::address}/${contrail::netmask_short}";
'VIRTUAL-HOST-INTERFACE/physical_interface': value => $contrail::phys_dev;
'SERVICE-INSTANCE/netns_command': value => '/usr/bin/opencontrail-vrouter-netns';
} ->
ini_setting { 'vrouter-threadcount':
'DEFAULT/platform': value => 'dpdk';
'DEFAULT/physical_interface_address' : value => $dpdk_dev_pci;
'DEFAULT/physical_interface_mac': value => $dpdk_dev_mac;
}
file {'/etc/network/interfaces.d/ifcfg-vhost0':
ensure => present,
path => '/etc/contrail/supervisord_vrouter.conf',
section => 'supervisord',
setting => 'environment',
value => 'TBB_THREAD_COUNT=8',
notify => Service['supervisor-vrouter'],
content => template('contrail/ubuntu-ifcfg-vhost0.erb'),
} ~>
service {'supervisor-vrouter':
ensure => running,
enable => true,
}
if $contrail::gateway {
contrail_vrouter_agent_config { 'VIRTUAL-HOST-INTERFACE/gateway': value => $contrail::gateway; }
}
if $contrail::compute_dpdk_enabled == true {
contrail_vrouter_agent_config {
'DEFAULT/platform': value => 'dpdk';
'DEFAULT/physical_interface_address' : value => $contrail::phys_dev_pci;
'DEFAULT/physical_interface_mac': value => $dpdk_dev_mac;
}
}
} else {
file {'/etc/network/interfaces.d/ifcfg-vhost0':
ensure => present,
content => template('contrail/ubuntu-ifcfg-vhost0.erb'),
} ~>
service {'supervisor-vrouter':
ensure => running,
enable => true,
@ -190,16 +186,7 @@ sync && \
echo 3 > /proc/sys/vm/drop_caches && \
echo 1 > /proc/sys/vm/compact_memory && \
service supervisor-vrouter start',
subscribe => [Package[$install_packages],Exec['remove-ovs-modules'],
File['/etc/contrail/agent_param']
],
}
# Temporary dirty hack. Network configuration fails because of deployed contrail vrouter [FIXME]
exec {'no_network_reconfigure':
command => '/bin/echo "#NOOP here. Modified by contrail plugin" > /etc/puppet/modules/osnailyfacter/modular/netconfig/netconfig.pp',
onlyif => '/usr/bin/test -f /opt/contrail/provision-vrouter-DONE',
}
}
Package[$install_packages] -> Contrail_vrouter_nodemgr_config <||> ~> Service['supervisor-vrouter']
Package[$install_packages] -> Contrail_vrouter_agent_config <||> ~> Service['supervisor-vrouter']

View File

@ -0,0 +1,45 @@
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
define contrail::create_vfs (
$totalvfs,
$numvfs,
$network_device = $title,
)
{
Exec {
provider => 'shell',
path => '/usr/bin:/bin:/sbin',
}
$final_vf = min(63, $totalvfs)
$interface_config = join(["auto ${network_device}",
"iface ${network_device} inet manual",
"post-up echo ${final_vf} > /sys/class/net/${network_device}/device/sriov_numvfs"
],"\n")
file {"/etc/network/interfaces.d/ifcfg-${network_device}":
ensure => file,
content => $interface_config,
}
exec {"create_vfs_on_${network_device}":
command => "echo ${final_vf} > /sys/class/net/${network_device}/device/sriov_numvfs",
tag => ['create_vfs'],
}
}

View File

@ -110,13 +110,12 @@ class contrail {
# DPDK settings
$global_dpdk_enabled = $settings['contrail_global_dpdk']
$dpdk_on_vf = $settings['dpdk_on_vf']
$compute_dpdk_enabled = $global_dpdk_enabled and 'dpdk' in hiera_array('roles')
$compute_dpdk_enabled = $global_dpdk_enabled and roles_include('dpdk')
# DPDK on VF settings
$compute_dpkd_on_vf = $compute_dpdk_enabled and $compute_sriov_enabled and $settings['dpdk_on_vf']
$dpdk_vf_number = 0
$compute_dpdk_on_vf = $compute_dpdk_enabled and roles_include('dpdk-on-vf')
$dpdk_physnet = $settings['dpdk_physnet']
$dpdk_vf_number = 0
# ToR/TSN agent settings
$enable_tor_agents = $settings['enable_tor_agents']

View File

@ -7,7 +7,7 @@ kmod=vrouter
pname=contrail-vrouter-agent
LIBDIR=/usr/lib64
DEVICE=vhost0
dev=<%= scope.lookupvar('contrail::phys_dev') %>
dev=<%= @phys_dev %>
vgw_subnet_ip=__VGW_SUBNET_IP__
vgw_intf=__VGW_INTF_LIST__
LOGFILE=--log-file=/var/log/contrail/vrouter.log

View File

@ -13,4 +13,7 @@ post-up ip route add <%= gw %>/32 via <%= scope.lookupvar('contrail::gateway') %
<% scope.lookupvar('contrail::routes').each do |route| -%>
post-up ip route add <%= route['net'] %> via <%= route['via'] %> dev vhost0
<% end -%>
<% end -%>
<% end -%>
<% if scope.lookupvar('contrail::compute_dpdk_enabled') -%>
post-up ip link set vhost0 address <%= @dpdk_dev_mac %>
<% end -%>

View File

@ -872,6 +872,17 @@
strategy:
type: one-by-one
- id: contrail-compute-dpdkonvf
type: puppet
version: 2.0.0
groups: [compute]
required_for: [contrail-compute-network,deploy_end]
requires: [top-role-compute, openstack-network-end]
parameters:
puppet_manifest: puppet/manifests/contrail-compute-dpdkonvf.pp
puppet_modules: puppet/modules:/etc/puppet/modules
timeout: 720
- id: contrail-compute-network
type: puppet
version: 2.0.0
@ -931,16 +942,6 @@
puppet_modules: puppet/modules:/etc/puppet/modules
timeout: 120
# Reboot compute nodes, if it need.
- id: compute-reboot
type: shell
role: [compute]
required_for: [post_deployment_end]
requires: [contrail-compute-aggregate]
parameters:
cmd: if [ -f /tmp/contrail-reboot-require ]; then nohup /sbin/shutdown -r +1 &> /dev/null & fi
timeout: 720
- id: sriov_iommu_check
type: skipped

View File

@ -93,16 +93,29 @@ attributes:
action: "hide"
dpdk_on_vf:
value: false
label: "DPDK on VF (experimental)"
label: "DPDK on VF"
description: "Configure DPDK vrouter use VF network adapter"
weight: 160
type: "checkbox"
restrictions:
- condition: "not (settings:contrail.metadata.enabled == true and settings:contrail.contrail_global_dpdk.value == true)"
action: "hide"
dpdk_physnet:
value: 'physnet2'
label: 'Provide name for physical net'
description: >
This physical network will be provided for SRIOV capable interfaces used with DPDK on VF.
weight: 220
type: 'text'
regex:
source: '^\w+$'
error: "Name has to be alphanumeric"
restrictions:
- action: hide
condition: not (settings:contrail.dpdk_on_vf.value == true)
enable_tor_agents:
value: false
label: "Enable ToR agents (experimental)"
label: "Enable ToR agents"
description: "Enable this option to unlock the Tor Agent configuration feature."
weight: 230
type: "checkbox"
@ -210,7 +223,7 @@ attributes:
action: "hide"
aaa_mode:
type: "select"
weight: 340
weight: 64
value: "cloud-admin"
label: "AAA Mode"
description: "Authentication mode"

View File

@ -133,10 +133,10 @@ dpdk:
- condition: "settings:contrail.contrail_global_dpdk.value == false"
action: hide
sriov:
name: SRIOV compute
dpdk-on-vf:
name: Compute with DPDK on VF
description: >
Enable SRIOV feature on compute node. NOTE: Use this role only together with Compute role to enable SRIOV feature in other cases it will not have any effect.
Enable DPDK on VF feature on compute node.
has_primary: false
public_ip_required: false
weight: 100
@ -146,6 +146,9 @@ sriov:
- contrail-config
- contrail-control
- contrail-analytics
restrictions:
- condition: "settings:contrail.dpdk_on_vf.value == false"
action: hide
contrail-vmware:
name: Contrail - VMware