openstack/openstack-helm
Code Issues Proposed changes

Extending Neutron chart with support for OVS DPDK

Browse Source 
Extending the Neutron with configuration parameters and scripts for
deploying OVS with DPDK support enabled. The new functionality takes
care of binding NICs to DPDK and adding those to OVS bridges of type
'netdev'.

Co-Authored-By: Rihab Banday <rihab.banday@ericsson.com>

Change-Id: I9932123986a0b723d7523136940d325bcfde983d
This commit is contained in:
Georg Kunz 2019-03-14 11:54:41 +01:00 committed by Rihab Banday
parent a66ecbf7ad
commit 1f8933da46
22 changed files with 311 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
neutron/templates/bin/_neutron-openvswitch-agent-init-modules.sh.tpl
View File

@ -18,3 +18,7 @@ limitations under the License.
set -ex
chroot /mnt/host-rootfs modprobe ip6_tables
{{- if .Values.conf.ovs_dpdk.enabled }}
chroot /mnt/host-rootfs modprobe {{ .Values.conf.ovs_dpdk.driver | quote }}
{{- end }}

165
neutron/templates/bin/_neutron-openvswitch-agent-init.sh.tpl
View File

@ -18,7 +18,162 @@ limitations under the License.
set -ex
chown neutron: /run/openvswitch/db.sock
OVS_SOCKET=/run/openvswitch/db.sock
chown neutron: ${OVS_SOCKET}
function get_dpdk_config_value {
values=$1
filter=$2
value=$(echo ${values} | jq -r ${filter})
if [[ "${value}" == "null" ]]; then
echo ""
else
echo "${value}"
fi
}
DPDK_CONFIG_FILE=/tmp/dpdk.conf
DPDK_CONFIG=""
DPDK_ENABLED=false
if [ -f ${DPDK_CONFIG_FILE} ]; then
DPDK_CONFIG=$(cat ${DPDK_CONFIG_FILE})
if [[ $(get_dpdk_config_value ${DPDK_CONFIG} '.enabled') == "true" ]]; then
DPDK_ENABLED=true
fi
fi
function bind_nic {
echo $2 > /sys/bus/pci/devices/$1/driver_override
echo $1 > /sys/bus/pci/drivers/$2/bind
}
function unbind_nic {
echo $1 > /sys/bus/pci/drivers/$2/unbind
echo > /sys/bus/pci/devices/$1/driver_override
}
function get_name_by_pci_id {
path=$(find /sys/bus/pci/devices/$1/ -name net)
if [ -n "${path}" ] ; then
echo $(ls -1 $path/)
fi
}
function get_ip_address_from_interface {
local interface=$1
local ip=$(ip -4 -o addr s "${interface}" | awk '{ print $4; exit }' | awk -F '/' '{print $1}')
if [ -z "${ip}" ] ; then
echo "Interface ${interface} has no valid IP address."
exit 1
fi
echo ${ip}
}
function get_ip_prefix_from_interface {
local interface=$1
local prefix=$(ip -4 -o addr s "${interface}" | awk '{ print $4; exit }' | awk -F '/' '{print $2}')
if [ -z "${prefix}" ] ; then
echo "Interface ${interface} has no valid IP address."
exit 1
fi
echo ${prefix}
}
function bind_dpdk_nics {
target_driver=$(get_dpdk_config_value ${DPDK_CONFIG} '.driver')
# loop over all nics
echo $DPDK_CONFIG | jq -r -c '.nics[]' | \
while IFS= read -r nic; do
local port_name=$(get_dpdk_config_value ${nic} '.name')
local pci_id=$(get_dpdk_config_value ${nic} '.pci_id')
local bridge=$(get_dpdk_config_value ${nic} '.bridge')
if [[ $(get_dpdk_config_value ${nic} '.migrate_ip') == true ]] ; then
local src_nic=$(get_name_by_pci_id ${pci_id})
if [ -n "${src_nic}" ] ; then
ip=$(get_ip_address_from_interface ${src_nic})
prefix=$(get_ip_prefix_from_interface ${src_nic})
# Enabling explicit error handling: We must avoid to lose the IP
# address in the migration process. Hence, on every error, we
# attempt to assign the IP back to the original NIC and exit.
set +e
ip addr flush dev ${src_nic}
if [ $? -ne 0 ] ; then
ip addr add ${ip}/${prefix} dev ${src_nic}
echo "Error while flushing IP from ${src_nic}."
exit 1
fi
bridge_exists=$(ip a s ${bridge} 2> /dev/null)
if [ -z "${bridge_exists}" ] ; then
echo "Bridge ${bridge} does not exist. Creating it on demand."
init_ovs_dpdk_bridge ${bridge}
fi
ip addr add ${ip}/${prefix} dev ${bridge}
if [ $? -ne 0 ] ; then
echo "Error assigning IP to bridge ${bridge}."
ip addr add ${ip}/${prefix} dev ${src_nic}
exit 1
fi
set -e
fi
fi
current_driver="$(get_driver_by_address ${pci_id} )"
if [ "$current_driver" != "$target_driver" ]; then
if [ "$current_driver" != "" ]; then
unbind_nic ${pci_id} ${current_driver}
fi
bind_nic ${pci_id} ${target_driver}
fi
ovs-vsctl --db=unix:${OVS_SOCKET} --if-exists del-port ${port_name}
dpdk_options=""
ofport_request=$(get_dpdk_config_value ${nic} '.ofport_request')
if [ -n "${ofport_request}" ]; then
dpdk_options+='ofport_request=${ofport_request} '
fi
n_rxq=$(get_dpdk_config_value ${nic} '.n_rxq')
if [ -n "${n_rxq}" ]; then
dpdk_options+='options:n_rxq=${n_rxq} '
fi
pmd_rxq_affinity=$(get_dpdk_config_value ${nic} '.pmd_rxq_affinity')
if [ -n "${pmd_rxq_affinity}" ]; then
dpdk_options+='other_config:pmd-rxq-affinity=${pmd_rxq_affinity} '
fi
ovs-vsctl --db=unix:${OVS_SOCKET} --may-exist add-port ${bridge} ${port_name} \
-- set Interface ${port_name} type=dpdk options:dpdk-devargs=${pci_id} ${dpdk_options}
done
}
function get_driver_by_address {
if [[ -e /sys/bus/pci/devices/$1/driver ]]; then
echo $(ls /sys/bus/pci/devices/$1/driver -al | awk '{n=split($NF,a,"/"); print a[n]}')
fi
}
function init_ovs_dpdk_bridge {
bridge=$1
ovs-vsctl --db=unix:${OVS_SOCKET} --may-exist add-br ${bridge} \
-- set Bridge ${bridge} datapath_type=netdev
ip link set ${bridge} up
}
# create all additional bridges defined in the DPDK section
function init_ovs_dpdk_bridges {
for br in $(get_dpdk_config_value ${DPDK_CONFIG} '.bridges[].name'); do
init_ovs_dpdk_bridge ${br}
done
}
# FIXME(portdirect): There is a neutron bug in Queens that needs resolved
# for now, if we cannot even get the version of neutron-sanity-check, skip
@ -61,8 +216,13 @@ if [ -z "${tunnel_interface}" ] ; then
| awk '{ print $1 }') || exit 1
fi
if [[ "${DPDK_ENABLED}" == "true" ]]; then
init_ovs_dpdk_bridges
bind_dpdk_nics
fi
# determine local-ip dynamically based on interface provided but only if tunnel_types is not null
LOCAL_IP=$(ip a s $tunnel_interface | grep 'inet ' | awk '{print $2}' | awk -F "/" '{print $1}')
LOCAL_IP=$(get_ip_address_from_interface ${tunnel_interface})
if [ -z "${LOCAL_IP}" ] ; then
echo "Var LOCAL_IP is empty"
exit 1
@ -72,3 +232,4 @@ tee > /tmp/pod-shared/ml2-local-ip.ini << EOF
[ovs]
local_ip = "${LOCAL_IP}"
EOF

9
neutron/templates/configmap-etc.yaml
View File

@ -169,6 +169,14 @@ just set it along with nova_metadata_host.
{{- $formatter_fluent := dict "class" "oslo_log.formatters.FluentFormatter" -}}
{{- $_ := set .Values.conf.logging "formatter_fluent" $formatter_fluent -}}
{{- end -}}
{{- if .Values.conf.ovs_dpdk.enabled -}}
{{- $_ := set $envAll.Values.conf.plugins.openvswitch_agent.ovs "datapath_type" "netdev" -}}
{{- if empty $envAll.Values.conf.plugins.openvswitch_agent.ovs.vhostuser_socket_dir -}}
{{- $_ := set $envAll.Values.conf.plugins.openvswitch_agent.ovs "vhostuser_socket_dir" "/run/openvswitch/vhostuser" -}}
{{- end -}}
{{- end -}}
---
apiVersion: v1
kind: Secret
@ -201,6 +209,7 @@ data:
neutron_sudoers: {{ $envAll.Values.conf.neutron_sudoers | b64enc }}
rootwrap.conf: {{ $envAll.Values.conf.rootwrap | b64enc }}
auto_bridge_add: {{ toJson $envAll.Values.conf.auto_bridge_add | b64enc }}
dpdk.conf: {{ toJson $envAll.Values.conf.ovs_dpdk | b64enc }}
{{- range $key, $value := $envAll.Values.conf.rootwrap_filters }}
{{- $filePrefix := replace "_" "-" $key }}
{{ printf "%s.filters" $filePrefix }}: {{ $value.content | b64enc }}

16
neutron/templates/daemonset-ovs-agent.yaml
View File

@ -102,6 +102,10 @@ spec:
- name: host-rootfs
mountPath: /mnt/host-rootfs
readOnly: true
{{- if .Values.conf.ovs_dpdk.enabled }}
- name: pci-devices
mountPath: /sys/bus/pci/devices
{{- end }}
- name: neutron-ovs-agent-init
{{ tuple $envAll "neutron_openvswitch_agent" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.agent.ovs | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
@ -158,6 +162,12 @@ spec:
mountPath: /etc/neutron/rootwrap.conf
subPath: rootwrap.conf
readOnly: true
{{- if .Values.conf.ovs_dpdk.enabled }}
- name: neutron-etc
mountPath: /tmp/dpdk.conf
subPath: dpdk.conf
readOnly: true
{{- end }}
{{- range $key, $value := $envAll.Values.conf.rootwrap_filters }}
{{- if ( has "ovs_agent" $value.pods ) }}
{{- if and ( eq "taas" $key ) (not $envAll.Values.conf.plugins.taas.taas.enabled) }}
@ -271,6 +281,12 @@ spec:
- name: host-rootfs
hostPath:
path: /
{{- if .Values.conf.ovs_dpdk.enabled }}
- name: pci-devices
hostPath:
path: /sys/bus/pci/devices
type: Directory
{{- end }}
{{ if $mounts_neutron_ovs_agent.volumes }}{{ toYaml $mounts_neutron_ovs_agent.volumes | indent 8 }}{{ end }}
{{- end }}
{{- end }}

17
neutron/values.yaml
View File

@ -1945,6 +1945,23 @@ conf:
auto_bridge_add:
br-ex: null
# configuration of OVS DPDK bridges and NICs
# this is a separate section and not part of the auto_bridge_add section
# because additional parameters are needed
ovs_dpdk:
enabled: false
# driver: uio_pci_generic
# nics:
# - name: dpdk0
# pci_id: '0000:05:00.0'
# bridge: br-phy
# migrate_ip: true
# n_rxq: 2
# pmd_rxq_affinity: "0:3,1:27"
# ofport_request: 1
# bridges:
# - name: br-phy
# Names of secrets used by bootstrap and environmental checks
secrets:
identity:

23
neutron/values_overrides/dpdk.yaml Normal file
View File

@ -0,0 +1,23 @@
network:
interface:
tunnel: br-phy
conf:
plugins:
openvswitch_agent:
agent:
tunnel_types: vxlan
ovs:
bridge_mappings: public:br-ex
datapath_type: netdev
vhostuser_socket_dir: /var/run/openvswitch/vhostuser
ovs_dpdk:
enabled: true
driver: uio_pci_generic
nics:
# CHANGE-ME: modify pci_id according to your hardware
- name: dpdk0
pci_id: '0000:05:00.0'
bridge: br-phy
migrate_ip: true
bridges:
- name: br-phy

22
neutron/values_overrides/gate.yaml Normal file
View File

@ -0,0 +1,22 @@
network:
interface:
tunnel: docker0
conf:
neutron:
DEFAULT:
l3_ha: False
max_l3_agents_per_router: 1
l3_ha_network_type: vxlan
dhcp_agents_per_network: 1
plugins:
ml2_conf:
ml2_type_flat:
flat_networks: public
openvswitch_agent:
agent:
tunnel_types: vxlan
ovs:
bridge_mappings: public:br-ex
linuxbridge_agent:
linux_bridge:
bridge_mappings: public:br-ex

1
tools/deployment/developer/dpdk/000-install-packages.sh Symbolic link
View File

@ -0,0 +1 @@
../../common/install-packages.sh

1
tools/deployment/developer/dpdk/010-deploy-k8s.sh Symbolic link
View File

@ -0,0 +1 @@
../../common/deploy-k8s.sh

1
tools/deployment/developer/dpdk/020-setup-client.sh Symbolic link
View File

@ -0,0 +1 @@
../../common/setup-client.sh

1
tools/deployment/developer/dpdk/030-ingress.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/common/ingress.sh

1
tools/deployment/developer/dpdk/040-nfs-provisioner.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/nfs-provisioner/nfs-provisioner.sh

1
tools/deployment/developer/dpdk/050-mariadb.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/common/mariadb.sh

1
tools/deployment/developer/dpdk/060-rabbitmq.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/common/rabbitmq.sh

1
tools/deployment/developer/dpdk/070-memcached.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/common/memcached.sh

1
tools/deployment/developer/dpdk/080-keystone.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/keystone/keystone.sh

1
tools/deployment/developer/dpdk/090-heat.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/heat/heat.sh

1
tools/deployment/developer/dpdk/100-horizon.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/horizon/horizon.sh

1
tools/deployment/developer/dpdk/120-glance.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/glance/glance.sh

22
tools/deployment/developer/dpdk/140-openvswitch.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
# enabling DPDK feature
export FEATURE_GATES="dpdk"
# Deploy openvswitch
./tools/deployment/component/compute-kit/openvswitch.sh

1
tools/deployment/developer/dpdk/150-libvirt.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/compute-kit/libvirt.sh

22
tools/deployment/developer/dpdk/160-compute-kit.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
# enabling DPDK feature
export FEATURE_GATES="dpdk,gate"
# Deploy nova and neutron charts
./tools/deployment/component/compute-kit/compute-kit.sh