openstack/openstack-helm
Code Issues Proposed changes

Merge "Extending Neutron chart with support for OVS DPDK"

Browse Source
This commit is contained in:
Zuul 2019-09-10 17:16:25 +00:00 committed by Gerrit Code Review
commit a4f84ad8fb
22 changed files with 311 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
neutron/templates/bin/_neutron-openvswitch-agent-init-modules.sh.tpl
View File

@ -18,3 +18,7 @@ limitations under the License.
set -ex
chroot /mnt/host-rootfs modprobe ip6_tables
{{- if .Values.conf.ovs_dpdk.enabled }}
chroot /mnt/host-rootfs modprobe {{ .Values.conf.ovs_dpdk.driver | quote }}
{{- end }}

165
neutron/templates/bin/_neutron-openvswitch-agent-init.sh.tpl
View File

@ -18,7 +18,162 @@ limitations under the License.
set -ex
chown neutron: /run/openvswitch/db.sock
OVS_SOCKET=/run/openvswitch/db.sock
chown neutron: ${OVS_SOCKET}
function get_dpdk_config_value {
values=$1
filter=$2
value=$(echo ${values} | jq -r ${filter})
if [[ "${value}" == "null" ]]; then
echo ""
else
echo "${value}"
fi
}
DPDK_CONFIG_FILE=/tmp/dpdk.conf
DPDK_CONFIG=""
DPDK_ENABLED=false
if [ -f ${DPDK_CONFIG_FILE} ]; then
DPDK_CONFIG=$(cat ${DPDK_CONFIG_FILE})
if [[ $(get_dpdk_config_value ${DPDK_CONFIG} '.enabled') == "true" ]]; then
DPDK_ENABLED=true
fi
fi
function bind_nic {
echo $2 > /sys/bus/pci/devices/$1/driver_override
echo $1 > /sys/bus/pci/drivers/$2/bind
}
function unbind_nic {
echo $1 > /sys/bus/pci/drivers/$2/unbind
echo > /sys/bus/pci/devices/$1/driver_override
}
function get_name_by_pci_id {
path=$(find /sys/bus/pci/devices/$1/ -name net)
if [ -n "${path}" ] ; then
echo $(ls -1 $path/)
fi
}
function get_ip_address_from_interface {
local interface=$1
local ip=$(ip -4 -o addr s "${interface}" | awk '{ print $4; exit }' | awk -F '/' '{print $1}')
if [ -z "${ip}" ] ; then
echo "Interface ${interface} has no valid IP address."
exit 1
fi
echo ${ip}
}
function get_ip_prefix_from_interface {
local interface=$1
local prefix=$(ip -4 -o addr s "${interface}" | awk '{ print $4; exit }' | awk -F '/' '{print $2}')
if [ -z "${prefix}" ] ; then
echo "Interface ${interface} has no valid IP address."
exit 1
fi
echo ${prefix}
}
function bind_dpdk_nics {
target_driver=$(get_dpdk_config_value ${DPDK_CONFIG} '.driver')
# loop over all nics
echo $DPDK_CONFIG | jq -r -c '.nics[]' | \
while IFS= read -r nic; do
local port_name=$(get_dpdk_config_value ${nic} '.name')
local pci_id=$(get_dpdk_config_value ${nic} '.pci_id')
local bridge=$(get_dpdk_config_value ${nic} '.bridge')
if [[ $(get_dpdk_config_value ${nic} '.migrate_ip') == true ]] ; then
local src_nic=$(get_name_by_pci_id ${pci_id})
if [ -n "${src_nic}" ] ; then
ip=$(get_ip_address_from_interface ${src_nic})
prefix=$(get_ip_prefix_from_interface ${src_nic})
# Enabling explicit error handling: We must avoid to lose the IP
# address in the migration process. Hence, on every error, we
# attempt to assign the IP back to the original NIC and exit.
set +e
ip addr flush dev ${src_nic}
if [ $? -ne 0 ] ; then
ip addr add ${ip}/${prefix} dev ${src_nic}
echo "Error while flushing IP from ${src_nic}."
exit 1
fi
bridge_exists=$(ip a s ${bridge} 2> /dev/null)
if [ -z "${bridge_exists}" ] ; then
echo "Bridge ${bridge} does not exist. Creating it on demand."
init_ovs_dpdk_bridge ${bridge}
fi
ip addr add ${ip}/${prefix} dev ${bridge}
if [ $? -ne 0 ] ; then
echo "Error assigning IP to bridge ${bridge}."
ip addr add ${ip}/${prefix} dev ${src_nic}
exit 1
fi
set -e
fi
fi
current_driver="$(get_driver_by_address ${pci_id} )"
if [ "$current_driver" != "$target_driver" ]; then
if [ "$current_driver" != "" ]; then
unbind_nic ${pci_id} ${current_driver}
fi
bind_nic ${pci_id} ${target_driver}
fi
ovs-vsctl --db=unix:${OVS_SOCKET} --if-exists del-port ${port_name}
dpdk_options=""
ofport_request=$(get_dpdk_config_value ${nic} '.ofport_request')
if [ -n "${ofport_request}" ]; then
dpdk_options+='ofport_request=${ofport_request} '
fi
n_rxq=$(get_dpdk_config_value ${nic} '.n_rxq')
if [ -n "${n_rxq}" ]; then
dpdk_options+='options:n_rxq=${n_rxq} '
fi
pmd_rxq_affinity=$(get_dpdk_config_value ${nic} '.pmd_rxq_affinity')
if [ -n "${pmd_rxq_affinity}" ]; then
dpdk_options+='other_config:pmd-rxq-affinity=${pmd_rxq_affinity} '
fi
ovs-vsctl --db=unix:${OVS_SOCKET} --may-exist add-port ${bridge} ${port_name} \
-- set Interface ${port_name} type=dpdk options:dpdk-devargs=${pci_id} ${dpdk_options}
done
}
function get_driver_by_address {
if [[ -e /sys/bus/pci/devices/$1/driver ]]; then
echo $(ls /sys/bus/pci/devices/$1/driver -al | awk '{n=split($NF,a,"/"); print a[n]}')
fi
}
function init_ovs_dpdk_bridge {
bridge=$1
ovs-vsctl --db=unix:${OVS_SOCKET} --may-exist add-br ${bridge} \
-- set Bridge ${bridge} datapath_type=netdev
ip link set ${bridge} up
}
# create all additional bridges defined in the DPDK section
function init_ovs_dpdk_bridges {
for br in $(get_dpdk_config_value ${DPDK_CONFIG} '.bridges[].name'); do
init_ovs_dpdk_bridge ${br}
done
}
# FIXME(portdirect): There is a neutron bug in Queens that needs resolved
# for now, if we cannot even get the version of neutron-sanity-check, skip
@ -61,8 +216,13 @@ if [ -z "${tunnel_interface}" ] ; then
| awk '{ print $1 }') || exit 1
fi
if [[ "${DPDK_ENABLED}" == "true" ]]; then
init_ovs_dpdk_bridges
bind_dpdk_nics
fi
# determine local-ip dynamically based on interface provided but only if tunnel_types is not null
LOCAL_IP=$(ip a s $tunnel_interface | grep 'inet ' | awk '{print $2}' | awk -F "/" '{print $1}')
LOCAL_IP=$(get_ip_address_from_interface ${tunnel_interface})
if [ -z "${LOCAL_IP}" ] ; then
echo "Var LOCAL_IP is empty"
exit 1
@ -72,3 +232,4 @@ tee > /tmp/pod-shared/ml2-local-ip.ini << EOF
[ovs]
local_ip = "${LOCAL_IP}"
EOF

9
neutron/templates/configmap-etc.yaml
View File

@ -169,6 +169,14 @@ just set it along with nova_metadata_host.
{{- $formatter_fluent := dict "class" "oslo_log.formatters.FluentFormatter" -}}
{{- $_ := set .Values.conf.logging "formatter_fluent" $formatter_fluent -}}
{{- end -}}
{{- if .Values.conf.ovs_dpdk.enabled -}}
{{- $_ := set $envAll.Values.conf.plugins.openvswitch_agent.ovs "datapath_type" "netdev" -}}
{{- if empty $envAll.Values.conf.plugins.openvswitch_agent.ovs.vhostuser_socket_dir -}}
{{- $_ := set $envAll.Values.conf.plugins.openvswitch_agent.ovs "vhostuser_socket_dir" "/run/openvswitch/vhostuser" -}}
{{- end -}}
{{- end -}}
---
apiVersion: v1
kind: Secret
@ -201,6 +209,7 @@ data:
neutron_sudoers: {{ $envAll.Values.conf.neutron_sudoers | b64enc }}
rootwrap.conf: {{ $envAll.Values.conf.rootwrap | b64enc }}
auto_bridge_add: {{ toJson $envAll.Values.conf.auto_bridge_add | b64enc }}
dpdk.conf: {{ toJson $envAll.Values.conf.ovs_dpdk | b64enc }}
{{- range $key, $value := $envAll.Values.conf.rootwrap_filters }}
{{- $filePrefix := replace "_" "-" $key }}
{{ printf "%s.filters" $filePrefix }}: {{ $value.content | b64enc }}

16
neutron/templates/daemonset-ovs-agent.yaml
View File

@ -102,6 +102,10 @@ spec:
- name: host-rootfs
mountPath: /mnt/host-rootfs
readOnly: true
{{- if .Values.conf.ovs_dpdk.enabled }}
- name: pci-devices
mountPath: /sys/bus/pci/devices
{{- end }}
- name: neutron-ovs-agent-init
{{ tuple $envAll "neutron_openvswitch_agent" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.agent.ovs | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
@ -158,6 +162,12 @@ spec:
mountPath: /etc/neutron/rootwrap.conf
subPath: rootwrap.conf
readOnly: true
{{- if .Values.conf.ovs_dpdk.enabled }}
- name: neutron-etc
mountPath: /tmp/dpdk.conf
subPath: dpdk.conf
readOnly: true
{{- end }}
{{- range $key, $value := $envAll.Values.conf.rootwrap_filters }}
{{- if ( has "ovs_agent" $value.pods ) }}
{{- if and ( eq "taas" $key ) (not $envAll.Values.conf.plugins.taas.taas.enabled) }}
@ -271,6 +281,12 @@ spec:
- name: host-rootfs
hostPath:
path: /
{{- if .Values.conf.ovs_dpdk.enabled }}
- name: pci-devices
hostPath:
path: /sys/bus/pci/devices
type: Directory
{{- end }}
{{ if $mounts_neutron_ovs_agent.volumes }}{{ toYaml $mounts_neutron_ovs_agent.volumes | indent 8 }}{{ end }}
{{- end }}
{{- end }}

17
neutron/values.yaml
View File

@ -1945,6 +1945,23 @@ conf:
auto_bridge_add:
br-ex: null
# configuration of OVS DPDK bridges and NICs
# this is a separate section and not part of the auto_bridge_add section
# because additional parameters are needed
ovs_dpdk:
enabled: false
# driver: uio_pci_generic
# nics:
# - name: dpdk0
# pci_id: '0000:05:00.0'
# bridge: br-phy
# migrate_ip: true
# n_rxq: 2
# pmd_rxq_affinity: "0:3,1:27"
# ofport_request: 1
# bridges:
# - name: br-phy
# Names of secrets used by bootstrap and environmental checks
secrets:
identity:

23
neutron/values_overrides/dpdk.yaml Normal file
View File

@ -0,0 +1,23 @@
network:
interface:
tunnel: br-phy
conf:
plugins:
openvswitch_agent:
agent:
tunnel_types: vxlan
ovs:
bridge_mappings: public:br-ex
datapath_type: netdev
vhostuser_socket_dir: /var/run/openvswitch/vhostuser
ovs_dpdk:
enabled: true
driver: uio_pci_generic
nics:
# CHANGE-ME: modify pci_id according to your hardware
- name: dpdk0
pci_id: '0000:05:00.0'
bridge: br-phy
migrate_ip: true
bridges:
- name: br-phy

22
neutron/values_overrides/gate.yaml Normal file
View File

@ -0,0 +1,22 @@
network:
interface:
tunnel: docker0
conf:
neutron:
DEFAULT:
l3_ha: False
max_l3_agents_per_router: 1
l3_ha_network_type: vxlan
dhcp_agents_per_network: 1
plugins:
ml2_conf:
ml2_type_flat:
flat_networks: public
openvswitch_agent:
agent:
tunnel_types: vxlan
ovs:
bridge_mappings: public:br-ex
linuxbridge_agent:
linux_bridge:
bridge_mappings: public:br-ex

1
tools/deployment/developer/dpdk/000-install-packages.sh Symbolic link
View File

@ -0,0 +1 @@
../../common/install-packages.sh

1
tools/deployment/developer/dpdk/010-deploy-k8s.sh Symbolic link
View File

@ -0,0 +1 @@
../../common/deploy-k8s.sh

1
tools/deployment/developer/dpdk/020-setup-client.sh Symbolic link
View File

@ -0,0 +1 @@
../../common/setup-client.sh

1
tools/deployment/developer/dpdk/030-ingress.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/common/ingress.sh

1
tools/deployment/developer/dpdk/040-nfs-provisioner.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/nfs-provisioner/nfs-provisioner.sh

1
tools/deployment/developer/dpdk/050-mariadb.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/common/mariadb.sh

1
tools/deployment/developer/dpdk/060-rabbitmq.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/common/rabbitmq.sh

1
tools/deployment/developer/dpdk/070-memcached.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/common/memcached.sh

1
tools/deployment/developer/dpdk/080-keystone.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/keystone/keystone.sh

1
tools/deployment/developer/dpdk/090-heat.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/heat/heat.sh

1
tools/deployment/developer/dpdk/100-horizon.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/horizon/horizon.sh

1
tools/deployment/developer/dpdk/120-glance.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/glance/glance.sh

22
tools/deployment/developer/dpdk/140-openvswitch.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
# enabling DPDK feature
export FEATURE_GATES="dpdk"
# Deploy openvswitch
./tools/deployment/component/compute-kit/openvswitch.sh

1
tools/deployment/developer/dpdk/150-libvirt.sh Symbolic link
View File

@ -0,0 +1 @@
../../component/compute-kit/libvirt.sh

22
tools/deployment/developer/dpdk/160-compute-kit.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
# enabling DPDK feature
export FEATURE_GATES="dpdk,gate"
# Deploy nova and neutron charts
./tools/deployment/component/compute-kit/compute-kit.sh