diff --git a/neutron/templates/bin/_neutron-openvswitch-agent-init-modules.sh.tpl b/neutron/templates/bin/_neutron-openvswitch-agent-init-modules.sh.tpl index 3321a9f7c3..83dc3f2018 100644 --- a/neutron/templates/bin/_neutron-openvswitch-agent-init-modules.sh.tpl +++ b/neutron/templates/bin/_neutron-openvswitch-agent-init-modules.sh.tpl @@ -18,3 +18,7 @@ limitations under the License. set -ex chroot /mnt/host-rootfs modprobe ip6_tables + +{{- if .Values.conf.ovs_dpdk.enabled }} +chroot /mnt/host-rootfs modprobe {{ .Values.conf.ovs_dpdk.driver | quote }} +{{- end }} diff --git a/neutron/templates/bin/_neutron-openvswitch-agent-init.sh.tpl b/neutron/templates/bin/_neutron-openvswitch-agent-init.sh.tpl index b9c4b78d65..ce4734da67 100644 --- a/neutron/templates/bin/_neutron-openvswitch-agent-init.sh.tpl +++ b/neutron/templates/bin/_neutron-openvswitch-agent-init.sh.tpl @@ -18,7 +18,162 @@ limitations under the License. set -ex -chown neutron: /run/openvswitch/db.sock +OVS_SOCKET=/run/openvswitch/db.sock +chown neutron: ${OVS_SOCKET} + + +function get_dpdk_config_value { + values=$1 + filter=$2 + value=$(echo ${values} | jq -r ${filter}) + if [[ "${value}" == "null" ]]; then + echo "" + else + echo "${value}" + fi +} + + +DPDK_CONFIG_FILE=/tmp/dpdk.conf +DPDK_CONFIG="" +DPDK_ENABLED=false +if [ -f ${DPDK_CONFIG_FILE} ]; then + DPDK_CONFIG=$(cat ${DPDK_CONFIG_FILE}) + if [[ $(get_dpdk_config_value ${DPDK_CONFIG} '.enabled') == "true" ]]; then + DPDK_ENABLED=true + fi +fi + +function bind_nic { + echo $2 > /sys/bus/pci/devices/$1/driver_override + echo $1 > /sys/bus/pci/drivers/$2/bind +} + +function unbind_nic { + echo $1 > /sys/bus/pci/drivers/$2/unbind + echo > /sys/bus/pci/devices/$1/driver_override +} + +function get_name_by_pci_id { + path=$(find /sys/bus/pci/devices/$1/ -name net) + if [ -n "${path}" ] ; then + echo $(ls -1 $path/) + fi +} + +function get_ip_address_from_interface { + local interface=$1 + local ip=$(ip -4 -o addr s "${interface}" | awk '{ print $4; exit }' | awk -F '/' '{print $1}') + if [ -z "${ip}" ] ; then + echo "Interface ${interface} has no valid IP address." + exit 1 + fi + echo ${ip} +} + +function get_ip_prefix_from_interface { + local interface=$1 + local prefix=$(ip -4 -o addr s "${interface}" | awk '{ print $4; exit }' | awk -F '/' '{print $2}') + if [ -z "${prefix}" ] ; then + echo "Interface ${interface} has no valid IP address." + exit 1 + fi + echo ${prefix} +} + +function bind_dpdk_nics { + target_driver=$(get_dpdk_config_value ${DPDK_CONFIG} '.driver') + + # loop over all nics + echo $DPDK_CONFIG | jq -r -c '.nics[]' | \ + while IFS= read -r nic; do + local port_name=$(get_dpdk_config_value ${nic} '.name') + local pci_id=$(get_dpdk_config_value ${nic} '.pci_id') + local bridge=$(get_dpdk_config_value ${nic} '.bridge') + + if [[ $(get_dpdk_config_value ${nic} '.migrate_ip') == true ]] ; then + local src_nic=$(get_name_by_pci_id ${pci_id}) + if [ -n "${src_nic}" ] ; then + ip=$(get_ip_address_from_interface ${src_nic}) + prefix=$(get_ip_prefix_from_interface ${src_nic}) + + # Enabling explicit error handling: We must avoid to lose the IP + # address in the migration process. Hence, on every error, we + # attempt to assign the IP back to the original NIC and exit. + set +e + ip addr flush dev ${src_nic} + if [ $? -ne 0 ] ; then + ip addr add ${ip}/${prefix} dev ${src_nic} + echo "Error while flushing IP from ${src_nic}." + exit 1 + fi + + bridge_exists=$(ip a s ${bridge} 2> /dev/null) + if [ -z "${bridge_exists}" ] ; then + echo "Bridge ${bridge} does not exist. Creating it on demand." + init_ovs_dpdk_bridge ${bridge} + fi + + ip addr add ${ip}/${prefix} dev ${bridge} + if [ $? -ne 0 ] ; then + echo "Error assigning IP to bridge ${bridge}." + ip addr add ${ip}/${prefix} dev ${src_nic} + exit 1 + fi + set -e + fi + fi + + current_driver="$(get_driver_by_address ${pci_id} )" + if [ "$current_driver" != "$target_driver" ]; then + if [ "$current_driver" != "" ]; then + unbind_nic ${pci_id} ${current_driver} + fi + bind_nic ${pci_id} ${target_driver} + fi + + ovs-vsctl --db=unix:${OVS_SOCKET} --if-exists del-port ${port_name} + + dpdk_options="" + ofport_request=$(get_dpdk_config_value ${nic} '.ofport_request') + if [ -n "${ofport_request}" ]; then + dpdk_options+='ofport_request=${ofport_request} ' + fi + n_rxq=$(get_dpdk_config_value ${nic} '.n_rxq') + if [ -n "${n_rxq}" ]; then + dpdk_options+='options:n_rxq=${n_rxq} ' + fi + pmd_rxq_affinity=$(get_dpdk_config_value ${nic} '.pmd_rxq_affinity') + if [ -n "${pmd_rxq_affinity}" ]; then + dpdk_options+='other_config:pmd-rxq-affinity=${pmd_rxq_affinity} ' + fi + + ovs-vsctl --db=unix:${OVS_SOCKET} --may-exist add-port ${bridge} ${port_name} \ + -- set Interface ${port_name} type=dpdk options:dpdk-devargs=${pci_id} ${dpdk_options} + + done +} + +function get_driver_by_address { + if [[ -e /sys/bus/pci/devices/$1/driver ]]; then + echo $(ls /sys/bus/pci/devices/$1/driver -al | awk '{n=split($NF,a,"/"); print a[n]}') + fi +} + +function init_ovs_dpdk_bridge { + bridge=$1 + ovs-vsctl --db=unix:${OVS_SOCKET} --may-exist add-br ${bridge} \ + -- set Bridge ${bridge} datapath_type=netdev + ip link set ${bridge} up +} + +# create all additional bridges defined in the DPDK section +function init_ovs_dpdk_bridges { + for br in $(get_dpdk_config_value ${DPDK_CONFIG} '.bridges[].name'); do + init_ovs_dpdk_bridge ${br} + done +} + # FIXME(portdirect): There is a neutron bug in Queens that needs resolved # for now, if we cannot even get the version of neutron-sanity-check, skip @@ -61,8 +216,13 @@ if [ -z "${tunnel_interface}" ] ; then | awk '{ print $1 }') || exit 1 fi +if [[ "${DPDK_ENABLED}" == "true" ]]; then + init_ovs_dpdk_bridges + bind_dpdk_nics +fi + # determine local-ip dynamically based on interface provided but only if tunnel_types is not null -LOCAL_IP=$(ip a s $tunnel_interface | grep 'inet ' | awk '{print $2}' | awk -F "/" '{print $1}') +LOCAL_IP=$(get_ip_address_from_interface ${tunnel_interface}) if [ -z "${LOCAL_IP}" ] ; then echo "Var LOCAL_IP is empty" exit 1 @@ -72,3 +232,4 @@ tee > /tmp/pod-shared/ml2-local-ip.ini << EOF [ovs] local_ip = "${LOCAL_IP}" EOF + diff --git a/neutron/templates/configmap-etc.yaml b/neutron/templates/configmap-etc.yaml index b07f16fb05..8dd37d2fd1 100644 --- a/neutron/templates/configmap-etc.yaml +++ b/neutron/templates/configmap-etc.yaml @@ -169,6 +169,14 @@ just set it along with nova_metadata_host. {{- $formatter_fluent := dict "class" "oslo_log.formatters.FluentFormatter" -}} {{- $_ := set .Values.conf.logging "formatter_fluent" $formatter_fluent -}} {{- end -}} + +{{- if .Values.conf.ovs_dpdk.enabled -}} +{{- $_ := set $envAll.Values.conf.plugins.openvswitch_agent.ovs "datapath_type" "netdev" -}} +{{- if empty $envAll.Values.conf.plugins.openvswitch_agent.ovs.vhostuser_socket_dir -}} +{{- $_ := set $envAll.Values.conf.plugins.openvswitch_agent.ovs "vhostuser_socket_dir" "/run/openvswitch/vhostuser" -}} +{{- end -}} +{{- end -}} + --- apiVersion: v1 kind: Secret @@ -201,6 +209,7 @@ data: neutron_sudoers: {{ $envAll.Values.conf.neutron_sudoers | b64enc }} rootwrap.conf: {{ $envAll.Values.conf.rootwrap | b64enc }} auto_bridge_add: {{ toJson $envAll.Values.conf.auto_bridge_add | b64enc }} + dpdk.conf: {{ toJson $envAll.Values.conf.ovs_dpdk | b64enc }} {{- range $key, $value := $envAll.Values.conf.rootwrap_filters }} {{- $filePrefix := replace "_" "-" $key }} {{ printf "%s.filters" $filePrefix }}: {{ $value.content | b64enc }} diff --git a/neutron/templates/daemonset-ovs-agent.yaml b/neutron/templates/daemonset-ovs-agent.yaml index 4a3e71de56..cd2e9001ce 100644 --- a/neutron/templates/daemonset-ovs-agent.yaml +++ b/neutron/templates/daemonset-ovs-agent.yaml @@ -102,6 +102,10 @@ spec: - name: host-rootfs mountPath: /mnt/host-rootfs readOnly: true +{{- if .Values.conf.ovs_dpdk.enabled }} + - name: pci-devices + mountPath: /sys/bus/pci/devices +{{- end }} - name: neutron-ovs-agent-init {{ tuple $envAll "neutron_openvswitch_agent" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.agent.ovs | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} @@ -158,6 +162,12 @@ spec: mountPath: /etc/neutron/rootwrap.conf subPath: rootwrap.conf readOnly: true + {{- if .Values.conf.ovs_dpdk.enabled }} + - name: neutron-etc + mountPath: /tmp/dpdk.conf + subPath: dpdk.conf + readOnly: true + {{- end }} {{- range $key, $value := $envAll.Values.conf.rootwrap_filters }} {{- if ( has "ovs_agent" $value.pods ) }} {{- if and ( eq "taas" $key ) (not $envAll.Values.conf.plugins.taas.taas.enabled) }} @@ -271,6 +281,12 @@ spec: - name: host-rootfs hostPath: path: / +{{- if .Values.conf.ovs_dpdk.enabled }} + - name: pci-devices + hostPath: + path: /sys/bus/pci/devices + type: Directory +{{- end }} {{ if $mounts_neutron_ovs_agent.volumes }}{{ toYaml $mounts_neutron_ovs_agent.volumes | indent 8 }}{{ end }} {{- end }} {{- end }} diff --git a/neutron/values.yaml b/neutron/values.yaml index 6c04b9fcb1..14b03f0a29 100644 --- a/neutron/values.yaml +++ b/neutron/values.yaml @@ -1945,6 +1945,23 @@ conf: auto_bridge_add: br-ex: null + # configuration of OVS DPDK bridges and NICs + # this is a separate section and not part of the auto_bridge_add section + # because additional parameters are needed + ovs_dpdk: + enabled: false + # driver: uio_pci_generic + # nics: + # - name: dpdk0 + # pci_id: '0000:05:00.0' + # bridge: br-phy + # migrate_ip: true + # n_rxq: 2 + # pmd_rxq_affinity: "0:3,1:27" + # ofport_request: 1 + # bridges: + # - name: br-phy + # Names of secrets used by bootstrap and environmental checks secrets: identity: diff --git a/neutron/values_overrides/dpdk.yaml b/neutron/values_overrides/dpdk.yaml new file mode 100644 index 0000000000..ff1a5419cb --- /dev/null +++ b/neutron/values_overrides/dpdk.yaml @@ -0,0 +1,23 @@ +network: + interface: + tunnel: br-phy +conf: + plugins: + openvswitch_agent: + agent: + tunnel_types: vxlan + ovs: + bridge_mappings: public:br-ex + datapath_type: netdev + vhostuser_socket_dir: /var/run/openvswitch/vhostuser + ovs_dpdk: + enabled: true + driver: uio_pci_generic + nics: + # CHANGE-ME: modify pci_id according to your hardware + - name: dpdk0 + pci_id: '0000:05:00.0' + bridge: br-phy + migrate_ip: true + bridges: + - name: br-phy diff --git a/neutron/values_overrides/gate.yaml b/neutron/values_overrides/gate.yaml new file mode 100644 index 0000000000..5f812791af --- /dev/null +++ b/neutron/values_overrides/gate.yaml @@ -0,0 +1,22 @@ +network: + interface: + tunnel: docker0 +conf: + neutron: + DEFAULT: + l3_ha: False + max_l3_agents_per_router: 1 + l3_ha_network_type: vxlan + dhcp_agents_per_network: 1 + plugins: + ml2_conf: + ml2_type_flat: + flat_networks: public + openvswitch_agent: + agent: + tunnel_types: vxlan + ovs: + bridge_mappings: public:br-ex + linuxbridge_agent: + linux_bridge: + bridge_mappings: public:br-ex diff --git a/tools/deployment/developer/dpdk/000-install-packages.sh b/tools/deployment/developer/dpdk/000-install-packages.sh new file mode 120000 index 0000000000..8ad28d13b0 --- /dev/null +++ b/tools/deployment/developer/dpdk/000-install-packages.sh @@ -0,0 +1 @@ +../../common/install-packages.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/010-deploy-k8s.sh b/tools/deployment/developer/dpdk/010-deploy-k8s.sh new file mode 120000 index 0000000000..3cb1bb90fa --- /dev/null +++ b/tools/deployment/developer/dpdk/010-deploy-k8s.sh @@ -0,0 +1 @@ +../../common/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/020-setup-client.sh b/tools/deployment/developer/dpdk/020-setup-client.sh new file mode 120000 index 0000000000..5ba93361cd --- /dev/null +++ b/tools/deployment/developer/dpdk/020-setup-client.sh @@ -0,0 +1 @@ +../../common/setup-client.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/030-ingress.sh b/tools/deployment/developer/dpdk/030-ingress.sh new file mode 120000 index 0000000000..7097683624 --- /dev/null +++ b/tools/deployment/developer/dpdk/030-ingress.sh @@ -0,0 +1 @@ +../../component/common/ingress.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/040-nfs-provisioner.sh b/tools/deployment/developer/dpdk/040-nfs-provisioner.sh new file mode 120000 index 0000000000..cb2307f8df --- /dev/null +++ b/tools/deployment/developer/dpdk/040-nfs-provisioner.sh @@ -0,0 +1 @@ +../../component/nfs-provisioner/nfs-provisioner.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/050-mariadb.sh b/tools/deployment/developer/dpdk/050-mariadb.sh new file mode 120000 index 0000000000..4e0232dbcc --- /dev/null +++ b/tools/deployment/developer/dpdk/050-mariadb.sh @@ -0,0 +1 @@ +../../component/common/mariadb.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/060-rabbitmq.sh b/tools/deployment/developer/dpdk/060-rabbitmq.sh new file mode 120000 index 0000000000..a4588fc2a1 --- /dev/null +++ b/tools/deployment/developer/dpdk/060-rabbitmq.sh @@ -0,0 +1 @@ +../../component/common/rabbitmq.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/070-memcached.sh b/tools/deployment/developer/dpdk/070-memcached.sh new file mode 120000 index 0000000000..cb6ef9a043 --- /dev/null +++ b/tools/deployment/developer/dpdk/070-memcached.sh @@ -0,0 +1 @@ +../../component/common/memcached.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/080-keystone.sh b/tools/deployment/developer/dpdk/080-keystone.sh new file mode 120000 index 0000000000..b48fa216c5 --- /dev/null +++ b/tools/deployment/developer/dpdk/080-keystone.sh @@ -0,0 +1 @@ +../../component/keystone/keystone.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/090-heat.sh b/tools/deployment/developer/dpdk/090-heat.sh new file mode 120000 index 0000000000..0566a0dfde --- /dev/null +++ b/tools/deployment/developer/dpdk/090-heat.sh @@ -0,0 +1 @@ +../../component/heat/heat.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/100-horizon.sh b/tools/deployment/developer/dpdk/100-horizon.sh new file mode 120000 index 0000000000..4d19e48dc3 --- /dev/null +++ b/tools/deployment/developer/dpdk/100-horizon.sh @@ -0,0 +1 @@ +../../component/horizon/horizon.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/120-glance.sh b/tools/deployment/developer/dpdk/120-glance.sh new file mode 120000 index 0000000000..dc8088d35c --- /dev/null +++ b/tools/deployment/developer/dpdk/120-glance.sh @@ -0,0 +1 @@ +../../component/glance/glance.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/140-openvswitch.sh b/tools/deployment/developer/dpdk/140-openvswitch.sh new file mode 100755 index 0000000000..7c0f4ee3e0 --- /dev/null +++ b/tools/deployment/developer/dpdk/140-openvswitch.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +set -xe + +# enabling DPDK feature +export FEATURE_GATES="dpdk" + +# Deploy openvswitch +./tools/deployment/component/compute-kit/openvswitch.sh diff --git a/tools/deployment/developer/dpdk/150-libvirt.sh b/tools/deployment/developer/dpdk/150-libvirt.sh new file mode 120000 index 0000000000..895091575a --- /dev/null +++ b/tools/deployment/developer/dpdk/150-libvirt.sh @@ -0,0 +1 @@ +../../component/compute-kit/libvirt.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/160-compute-kit.sh b/tools/deployment/developer/dpdk/160-compute-kit.sh new file mode 100755 index 0000000000..62e1707aec --- /dev/null +++ b/tools/deployment/developer/dpdk/160-compute-kit.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +set -xe + +# enabling DPDK feature +export FEATURE_GATES="dpdk,gate" + +# Deploy nova and neutron charts +./tools/deployment/component/compute-kit/compute-kit.sh