From 4f5bac108952a4f7e0bec3789bea175c5875d6f7 Mon Sep 17 00:00:00 2001 From: Sean Mooney Date: Wed, 29 Mar 2017 15:05:55 +0000 Subject: [PATCH] introduces support for the OVS DPDK dataplane - DPDK is a userspace library for high-performance networking. - This change enable support of Open vSwitch with the DPDK accelerated netdev datapath. - This change provided binary and source support via a template-override. - This change provides an example template override file and corresponding documentation. Co-Authored-By: Mauricio Lima Partial-Implements: blueprint ovs-dpdk Change-Id: I7c3a5dddeca1886fe1c7681ce8d5bebde08f3102 --- contrib/template-override/ovs-dpdk.j2 | 75 +++++++++++++++++++ doc/template-override/ovs-dpdk.rst | 31 ++++++++ docker/macros.j2 | 8 +- .../openvswitch-base/Dockerfile.j2 | 4 + docker/ovsdpdk/ovsdpdk-db/Dockerfile.j2 | 12 +++ docker/ovsdpdk/ovsdpdk-db/extend_start.sh | 6 ++ .../ovsdpdk/ovsdpdk-db/start_ovsdb_server.sh | 30 ++++++++ docker/ovsdpdk/ovsdpdk-vswitchd/Dockerfile.j2 | 10 +++ .../ovsdpdk/ovsdpdk-vswitchd/extend_start.sh | 6 ++ docker/ovsdpdk/ovsdpdk/Dockerfile.j2 | 35 +++++++++ docker/ovsdpdk/ovsdpdk/extend_start.sh | 10 +++ tests/test_build.py | 8 +- 12 files changed, 232 insertions(+), 3 deletions(-) create mode 100644 contrib/template-override/ovs-dpdk.j2 create mode 100644 doc/template-override/ovs-dpdk.rst create mode 100644 docker/ovsdpdk/ovsdpdk-db/Dockerfile.j2 create mode 100644 docker/ovsdpdk/ovsdpdk-db/extend_start.sh create mode 100755 docker/ovsdpdk/ovsdpdk-db/start_ovsdb_server.sh create mode 100644 docker/ovsdpdk/ovsdpdk-vswitchd/Dockerfile.j2 create mode 100644 docker/ovsdpdk/ovsdpdk-vswitchd/extend_start.sh create mode 100644 docker/ovsdpdk/ovsdpdk/Dockerfile.j2 create mode 100644 docker/ovsdpdk/ovsdpdk/extend_start.sh diff --git a/contrib/template-override/ovs-dpdk.j2 b/contrib/template-override/ovs-dpdk.j2 new file mode 100644 index 0000000000..7dea1b7fd2 --- /dev/null +++ b/contrib/template-override/ovs-dpdk.j2 @@ -0,0 +1,75 @@ +{% extends parent_template %} + +{% block ovsdpdk_install %} + +# install ovs from source + +{% if base_distro in ['centos', 'oraclelinux', 'rhel'] %} + {% set ovs_dpdk_packages = [ + 'kernel-devel-$(uname -r)', + 'redhat-lsb-core', + 'pciutils', + 'autoconf', + 'libtool', + 'fuse-devel', + 'gcc', + 'make', + 'autoconf', + 'automake', + 'libpcap-devel', + 'python-devel', + 'python-six', + 'crudini' + ] %} + +{% elif base_distro in [ 'debian', 'ubuntu' ] %} + + {% set ovs_dpdk_packages = [ + 'linux-headers-$(uname -r)', + 'fdutils', + 'libxtst6', + 'libnuma-dev', + 'autoconf', + 'automake', + 'libtool', + 'libfuse-dev', + 'gcc', + 'make', + 'build-essential', + 'libpcap-dev', + 'python-dev', + 'python-six', + 'crudini' + ] %} + +{% endif %} + +{{ macros.install_packages(ovs_dpdk_packages | customizable("packages")) }} + +ADD plugins-archive / +RUN ln -s /plugins/ovsdpdk-plugin-ovs-archive-*/ /ovs \ + && ln -s /plugins/ovsdpdk-plugin-dpdk-archive-*/ /dpdk +WORKDIR /dpdk +{% set RTE_TARGET='x86_64-native-linuxapp-gcc' %} +RUN make config T="{{ RTE_TARGET }}" +WORKDIR "/dpdk/build" + +{{ macros.sed( file="/dpdk/build/.config", + expressions=["s/CONFIG_RTE_BUILD_COMBINE_LIBS=n/CONFIG_RTE_BUILD_COMBINE_LIBS=y/", + "s/CONFIG_RTE_MAX_MEMSEG=.*$/CONFIG_RTE_MAX_MEMSEG=1024/", + "s/CONFIG_RTE_LIBRTE_VHOST=.*$/CONFIG_RTE_LIBRTE_VHOST=y/", + "s/CONFIG_RTE_LIBRTE_KNI=.*$/CONFIG_RTE_LIBRTE_KNI=n/", + "s/CONFIG_RTE_KNI_KMOD=.*$/CONFIG_RTE_KNI_KMOD=n/", + "s/CONFIG_RTE_EAL_IGB_UIO=.*$/CONFIG_RTE_EAL_IGB_UIO=y/", + "s/CONFIG_RTE_BUILD_SHARED_LIB=.*$/CONFIG_RTE_BUILD_SHARED_LIB=n/"]) +}} + +RUN make -j $(nproc) EXTRA_CFLAGS='-O2 -fPIC' \ + && make install +WORKDIR /ovs +RUN ./boot.sh \ + && ./configure --with-dpdk="/dpdk/build" --prefix=/usr --with-dbdir=/etc/openvswitch --with-rundir=/run/openvswitch \ + && make -j $(nproc) CFLAGS='-O2 -fPIC' \ + && make install + +{% endblock %} diff --git a/doc/template-override/ovs-dpdk.rst b/doc/template-override/ovs-dpdk.rst new file mode 100644 index 0000000000..e47cfe0ffe --- /dev/null +++ b/doc/template-override/ovs-dpdk.rst @@ -0,0 +1,31 @@ +OVS-DPDK Source build +===================== + +CentOS and Oracle Linux currently do not provide packages +for ovs with dpdk. +The Ubuntu packages do not support UIO based drivers. +To use the uio_pci_generic driver on Ubuntu a source build is required. + +Building ovs with dpdk containers from source +--------------------------------------------- + +- Append the following to /etc/kolla/kolla-build.conf to select the version + of ovs and dpdk to use for your source build. + +kolla-build.conf +________________ + +[openvswitch-base-plugin-ovs] +type = git +location = https://github.com/openvswitch/ovs.git +reference = v2.7.0 + +[openvswitch-base-plugin-dpdk] +type = git +location = http://dpdk.org/git/dpdk +reference = v17.02 + +- To build the container execute the follow command:: + + tools/build.py --template-override \ + contrib/template-override/ovs-dpdk.j2 dpdk diff --git a/docker/macros.j2 b/docker/macros.j2 index 9120b5142d..203ca4be53 100644 --- a/docker/macros.j2 +++ b/docker/macros.j2 @@ -13,6 +13,13 @@ {%- endif %} {%- endmacro %} +{% macro sed(file='', expressions=[]) -%} +{% if file != '' and expressions|length >0 %} + RUN {% for expression in expressions %} sed -i "{{ file }}" -e "{{ expression }}" {% if not loop.last %} && {% endif %} {% endfor %} +{% else %} + RUN echo "file and expressions must be set to use the sed macro"; /bin/false +{% endif %} +{%- endmacro %} {% macro install_pip(packages, constraints = true) %} {%- if packages is sequence and packages|length > 0 -%} @@ -24,7 +31,6 @@ {%- endif -%} {% endmacro %} - {% macro configure_user(name, groups=None, shell=None, homedir=None) %} {% set user=users[name] %} {%- if not homedir %} diff --git a/docker/openvswitch/openvswitch-base/Dockerfile.j2 b/docker/openvswitch/openvswitch-base/Dockerfile.j2 index 75c60d8903..1d16e88723 100644 --- a/docker/openvswitch/openvswitch-base/Dockerfile.j2 +++ b/docker/openvswitch/openvswitch-base/Dockerfile.j2 @@ -5,6 +5,8 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% import "macros.j2" as macros with context %} +{% block ovs_install %} + {% if base_distro in ['centos', 'oraclelinux', 'rhel'] %} {% set openvswitch_base_packages = ['openvswitch'] %} @@ -14,6 +16,8 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% endif %} {{ macros.install_packages(openvswitch_base_packages | customizable("packages")) }} +{% endblock %} + COPY extend_start.sh /usr/local/bin/kolla_extend_start RUN touch /usr/local/bin/kolla_openvswitch_extend_start \ diff --git a/docker/ovsdpdk/ovsdpdk-db/Dockerfile.j2 b/docker/ovsdpdk/ovsdpdk-db/Dockerfile.j2 new file mode 100644 index 0000000000..10840f2e81 --- /dev/null +++ b/docker/ovsdpdk/ovsdpdk-db/Dockerfile.j2 @@ -0,0 +1,12 @@ +FROM {{ namespace }}/{{ image_prefix }}ovsdpdk:{{ tag }} +LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}" + +{% block ovsdpdk_db_header %}{% endblock %} + +COPY extend_start.sh /usr/local/bin/kolla_ovsdpdk_extend_start +COPY start_ovsdb_server.sh /usr/local/bin/start-ovsdb-server +RUN chmod 755 /usr/local/bin/kolla_ovsdpdk_extend_start && \ + chmod 755 /usr/local/bin/start-ovsdb-server + +{% block ovsdpdk_db_footer %}{% endblock %} +{% block footer %}{% endblock %} diff --git a/docker/ovsdpdk/ovsdpdk-db/extend_start.sh b/docker/ovsdpdk/ovsdpdk-db/extend_start.sh new file mode 100644 index 0000000000..3883b50ed8 --- /dev/null +++ b/docker/ovsdpdk/ovsdpdk-db/extend_start.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +mkdir -p "/run/openvswitch" +if [[ ! -e "/var/lib/openvswitch/conf.db" ]]; then + ovsdb-tool create "/var/lib/openvswitch/conf.db" +fi diff --git a/docker/ovsdpdk/ovsdpdk-db/start_ovsdb_server.sh b/docker/ovsdpdk/ovsdpdk-db/start_ovsdb_server.sh new file mode 100755 index 0000000000..5c92fba781 --- /dev/null +++ b/docker/ovsdpdk/ovsdpdk-db/start_ovsdb_server.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# NOTE: (sbezverk) ovs_bridge and ovs_ext_intf variables get initialized only when +# this script is executed for kubernetes deployment. With Ansible deployment, only +# ovsdb-server gets launched and then the following workflow step will create +# an external bridge and plug an external interface. With Kubernetes we want to +# leverage its dynamic nature of automatic scaling up and down. It means all +# activities related to creating initial bridge, plugging external interface +# must be done by DaemonSet launched container. + +ovsdb_ip=$1 +ovs_bridge=$2 +ovs_ext_intf=$3 + +# NOTE: (sbezverk) The reason for introducing this script is to be able +# to launch ovsdb-server and to create the initial external bridge in one step. +# It is required in order to be able to use DaemonSet. + +if [ ! -e $ovs_bridge ] && [ ! -e $ovs_ext_intf ]; then +# NOTE: (sbezverk) This part is executed only by kubernetes deployment. +# Creating external bridge + /usr/sbin/ovsdb-server /var/lib/openvswitch/conf.db --remote=punix:/var/run/openvswitch/db.sock --run="ovs-vsctl --no-wait --db=unix:/var/run/openvswitch/db.sock add-br $ovs_bridge" +# Plug the external interface into the external bridge. + /usr/sbin/ovsdb-server /var/lib/openvswitch/conf.db --remote=punix:/var/run/openvswitch/db.sock --run="ovs-vsctl --no-wait --db=unix:/var/run/openvswitch/db.sock add-port $ovs_bridge $ovs_ext_intf" +# Run ovsdb server process + /usr/sbin/ovsdb-server /var/lib/openvswitch/conf.db -vconsole:emer -vsyslog:err -vfile:info --remote=punix:/var/run/openvswitch/db.sock --remote=ptcp:6640 --log-file=/var/log/kolla/openvswitch/ovsdb-server.log +else +# NOTE: (sbezverk) This part is executed only by kolla-ansible deployment. + /usr/sbin/ovsdb-server /var/lib/openvswitch/conf.db -vconsole:emer -vsyslog:err -vfile:info --remote=punix:/run/openvswitch/db.sock --remote=ptcp:6640:$ovsdb_ip --log-file=/var/log/kolla/openvswitch/ovsdb-server.log +fi diff --git a/docker/ovsdpdk/ovsdpdk-vswitchd/Dockerfile.j2 b/docker/ovsdpdk/ovsdpdk-vswitchd/Dockerfile.j2 new file mode 100644 index 0000000000..3e71d1d732 --- /dev/null +++ b/docker/ovsdpdk/ovsdpdk-vswitchd/Dockerfile.j2 @@ -0,0 +1,10 @@ +FROM {{ namespace }}/{{ image_prefix }}ovsdpdk:{{ tag }} +LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}" + +{% block ovsdpdk_vswitchd_header %}{% endblock %} + +COPY extend_start.sh /usr/local/bin/kolla_ovsdpdk_extend_start +RUN chmod 755 /usr/local/bin/kolla_ovsdpdk_extend_start + +{% block ovsdpdk_vswitchd_footer %}{% endblock %} +{% block footer %}{% endblock %} diff --git a/docker/ovsdpdk/ovsdpdk-vswitchd/extend_start.sh b/docker/ovsdpdk/ovsdpdk-vswitchd/extend_start.sh new file mode 100644 index 0000000000..cf9d733e8c --- /dev/null +++ b/docker/ovsdpdk/ovsdpdk-vswitchd/extend_start.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# using userspace netdev datapath so do not loading ovs kernel module +# chmod openvsitch run directory so libvirt can create vhost-user sockets. +chmod 777 /var/run/openvswitch + diff --git a/docker/ovsdpdk/ovsdpdk/Dockerfile.j2 b/docker/ovsdpdk/ovsdpdk/Dockerfile.j2 new file mode 100644 index 0000000000..d64e667181 --- /dev/null +++ b/docker/ovsdpdk/ovsdpdk/Dockerfile.j2 @@ -0,0 +1,35 @@ +FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }} +LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}" + +{% import "macros.j2" as macros with context %} +{% block ovsdpdk_header %}{% endblock %} + +{% block ovsdpdk_install %} + +{% if base_distro in ['centos', 'oraclelinux', 'rhel'] %} + +#TODO(sean-k-mooney) create repo via open suse build system to package +# for centos/oraclelinux/rhel distros. + +RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \ + && /bin/false + +{% elif base_distro in ['ubuntu', 'debian'] %} + +{% set ovs_dpdk_packages = [ + 'openvswitch-switch-dpdk' +] %} + +{{ macros.install_packages(ovs_dpdk_packages | customizable("packages")) }} + +RUN update-alternatives --set ovs-vswitchd /usr/lib/openvswitch-switch-dpdk/ovs-vswitchd-dpdk + +{% endif %} + +{% endblock %} + +COPY extend_start.sh /usr/local/bin/kolla_extend_start +RUN chmod 755 /usr/local/bin/kolla_extend_start + +{% block ovsdpdk_footer %}{% endblock %} +{% block footer %}{% endblock %} diff --git a/docker/ovsdpdk/ovsdpdk/extend_start.sh b/docker/ovsdpdk/ovsdpdk/extend_start.sh new file mode 100644 index 0000000000..60ab0c0880 --- /dev/null +++ b/docker/ovsdpdk/ovsdpdk/extend_start.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +if [[ ! -d "/var/log/kolla/openvswitch" ]]; then + mkdir -p /var/log/kolla/openvswitch +fi +if [[ $(stat -c %a /var/log/kolla/openvswitch) != "755" ]]; then + chmod 755 /var/log/kolla/openvswitch +fi + +. /usr/local/bin/kolla_ovsdpdk_extend_start diff --git a/tests/test_build.py b/tests/test_build.py index c138ae34a1..0036759843 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -81,6 +81,7 @@ class BuildTestCentosBinary(BuildTest, base.BaseTestCase): "kuryr-base", "monasca-base", "neutron-bgp-dragent", + "ovsdpdk", "searchlight-base", "senlin-base", "solum-base", @@ -100,7 +101,8 @@ class BuildTestCentosSource(BuildTest, base.BaseTestCase): "bifrost-base", "kafka", "mistral-base", - "opendaylight" + "opendaylight", + "ovsdpdk", ] def setUp(self): @@ -207,6 +209,7 @@ class BuildTestOracleLinuxBinary(BuildTest, base.BaseTestCase): "kuryr-base", "monasca-base", "neutron-bgp-dragent", + "ovsdpdk", "searchlight-base", "senlin-base", "solum-base", @@ -225,7 +228,8 @@ class BuildTestOracleLinuxSource(BuildTest, base.BaseTestCase): excluded_images = [ "bifrost-base", "kafka", - "opendaylight" + "opendaylight", + "ovsdpdk", ] def setUp(self):