Browse Source

Initial DPDK support

Adding initial support for OVS DPDK.

Signed-off-by: Gabor Meszaros <gabor@meszaros.pw>
Change-Id: I1f87188bd8b4b928108674a35e9a9c309e9f6d17
changes/35/693035/29
Gabor Meszaros 2 months ago
parent
commit
ce5e82e319
11 changed files with 398 additions and 57 deletions
  1. +1
    -0
      snap-overlay/bin/set-default-config
  2. +5
    -3
      snap-overlay/bin/setup-br-ex
  3. +70
    -0
      snap-wrappers/ovs/ovs-alternatives
  4. +30
    -4
      snap-wrappers/ovs/ovs-wrapper
  5. +3
    -0
      snap/hooks/configure
  6. +3
    -0
      snap/hooks/install
  7. +5
    -0
      snap/hooks/remove
  8. +133
    -2
      snapcraft.yaml
  9. +82
    -45
      tools/init/init/questions/__init__.py
  10. +61
    -1
      tools/init/init/questions/network.py
  11. +5
    -2
      tools/init/init/shell.py

+ 1
- 0
snap-overlay/bin/set-default-config View File

@@ -23,6 +23,7 @@ snapctl set \
config.network.ports.dashboard=80 \
config.network.ports.mysql=3306 \
config.network.ports.rabbit=5672 \
config.network.ovs-dpdk=false \
;

# Passwords, certs, etc.

+ 5
- 3
snap-overlay/bin/setup-br-ex View File

@@ -12,13 +12,15 @@ set -ex
extcidr=$(snapctl get config.network.ext-cidr)

# Create external integration bridge
ovs-vsctl --retry --may-exist add-br br-ex
ovs-wrapper ovs-vsctl --retry --may-exist add-br br-ex
[ $(snapctl get config.network.ovs-dpdk) == true ] && \
ovs-wrapper ovs-vsctl -- set Bridge br-ex datapath_type=netdev

# Configure br-ex
ip address add $extcidr dev br-ex || :
ip link set br-ex up || :

sudo iptables -w -t nat -A POSTROUTING -s $extcidr ! \
-d $extcidr -j MASQUERADE || :
sudo iptables -w -t nat -C POSTROUTING -s $extcidr ! -d $extcidr -j MASQUERADE || \
sudo iptables -w -t nat -A POSTROUTING -s $extcidr ! -d $extcidr -j MASQUERADE

exit 0

+ 70
- 0
snap-wrappers/ovs/ovs-alternatives View File

@@ -0,0 +1,70 @@
#!/bin/bash

# this initializes the ovs commands alternatives with the default non-dpdk binaries
set -e

function _remove(){
#remove both openvswitch and openvswitch-dpdk alternatives of the current release.
# it is called at post-refresh hook time and at remove hook time also.
for version in openvswitch openvswitch-dpdk; do
for dir in \
bin \
etc/init.d \
sbin \
share/openvswitch/scripts \
usr/bin \
var/snap/${SNAP_NAME}/etc/bash_completion.d \
; do
cd ${SNAP}/${version}/${dir}/
for file in *; do
update-alternatives --quiet --remove ${file} ${SNAP}/${version}/${dir}/${file}
done
cd - &>/dev/null
done
done
}

function _install(){
# only set version alternatives when it is configured via snap config options
version="openvswitch"
if [[ $(snapctl get config.network.ovs-dpdk) == *"rue" ]]; then
version="openvswitch-dpdk"
fi

for dir in \
bin \
etc/init.d \
sbin \
share/openvswitch/scripts \
usr/bin \
var/snap/${SNAP_NAME}/etc/bash_completion.d \
; do
cd ${SNAP}/${version}/${dir}/
mkdir -p ${SNAP_COMMON}/${dir}/
for file in *; do
update-alternatives --quiet --install $SNAP_COMMON/$dir/${file} ${file} ${SNAP}/${version}/${dir}/${file} 1
update-alternatives --quiet --set ${file} ${SNAP}/${version}/${dir}/${file}
done
cd - &>/dev/null
done
}

while [[ $# -gt 0 ]]
do
key="$1"
case $key in
-i|--install)
shift # past argument
_install
;;
-r|--remove)
shift # past argument
_remove
;;
*) # unknown option, noop
shift # past argument
;;
esac
done

exit 0

+ 30
- 4
snap-wrappers/ovs/ovs-wrapper View File

@@ -1,16 +1,42 @@
#!/bin/bash

# important: not to echo or print anything on stdout, will be taken into ovs-ctl

set -e

export OVS_LOGDIR=${SNAP_COMMON}/log/openvswitch
export OVS_RUNDIR=${SNAP_COMMON}/run/openvswitch
export OVS_SYSCONFDIR=${SNAP_COMMON}/etc
export OVS_PKGDATADIR=${SNAP}/share/openvswitch
export OVS_BINDIR=${SNAP}/bin
export OVS_SBINDIR=${SNAP}/sbin
export OVS_PKGDATADIR=${SNAP}/openvswitch-dpdk/share/openvswitch

if [ "$(snapctl get config.network.ovs-dpdk)" == "false" ]; then
export OVS_BINDIR=${SNAP}/openvswitch/bin
export OVS_SBINDIR=${SNAP}/openvswitch/sbin
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${SNAP}/openvswitch/lib:${SNAP}/openvswitch/usr/lib/${SNAPCRAFT_ARCH_TRIPLET}"
else
export OVS_BINDIR=${SNAP}/openvswitch-dpdk/bin
export OVS_SBINDIR=${SNAP}/openvswitch-dpdk/sbin
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${SNAP}/openvswitch-dpdk/lib:${SNAP}/openvswitch-dpdk/usr/lib/${SNAPCRAFT_ARCH_TRIPLET}"
fi

mkdir -p ${OVS_LOGDIR}
mkdir -p ${OVS_RUNDIR}
mkdir -p ${OVS_SYSCONFDIR}/openvswitch

exec $@
# need to maintain ovsdb-server and ovs-vswtichd unix socket path:
shopt -s nullglob
for ctl in $OVS_RUNDIR/*.real.*.ctl;
do
ln -fs $ctl "${ctl//.real/}"
done
# needs further testing: vvv
#echo "find $OVS_RUNDIR/*.ctl -xtype l -delete"
shopt -u nullglob

if [[ "$0" == *"ovs-wrapper"* ]]; then
cmd=$(command -v ${1})
exec ${cmd%/*}/real/${cmd##*/} "${@:2}"
else
# command is symlink to this script
exec "${0%/*}/real/${0##*/}" "${@}"
fi

+ 3
- 0
snap/hooks/configure View File

@@ -1,6 +1,9 @@
#!/bin/bash
set -ex

# update the openvswitch binaries alternatives if needed
${SNAP}/bin/ovs-alternatives --install

snap-openstack setup # Write out templates

source $SNAP_COMMON/etc/microstack.rc

+ 3
- 0
snap/hooks/install View File

@@ -22,4 +22,7 @@ done
# Make a place for our horizon config overrides to live
mkdir -p ${SNAP_COMMON}/etc/horizon/local_settings.d

# provide initial openvswitch binaries alternatives
${SNAP}/bin/ovs-alternatives --install

snap-openstack setup # Sets up templates for the first time.

+ 5
- 0
snap/hooks/remove View File

@@ -0,0 +1,5 @@
#!/bin/bash
set -ex

# cleanup openvswitch binaries alternatives
${SNAP}/bin/ovs-alternatives --remove

+ 133
- 2
snapcraft.yaml View File

@@ -318,8 +318,69 @@ apps:
join:
command: python3 ${SNAP}/lib/python3.6/site-packages/cluster/client.py

testpmd:
command: dpdk/bin/testpmd
#plugs:
# - network
# - network-bind
# - network-control
# - network-observe
# - hardware-observe
# #- hugepages-observe
# #- hugepages-control
# - process-control
# - system-observe
# #- dpdk-control
# - kernel-module-control
# - kernel-module-observe

dpdk-devbind:
command: dpdk/sbin/dpdk-devbind
#plugs:
# - network
# - network-bind
# - network-control
# - network-observe
# - hardware-observe
# - system-observe
# #- dpdk-control
# - kernel-module-control
# - kernel-module-observe

dpdk-setup:
command: dpdk/share/dpdk/usertools/dpdk-setup.sh
#plugs:
# - network
# - network-bind
# - network-control
# - network-observe
# - hardware-observe
# #- system-observer
# #- dpdk-control
# - kernel-module-control
# - kernel-module-observe

ovs-alternatives:
command: bin/ovs-alternatives

parts:
dpdk:
plugin: make
source: http://dpdk.org/git/dpdk-stable
source-type: git
source-tag: v18.11 # LTS Version, comment out or change tag to what version you require.
after:
- patches
- uca-sources
build-packages:
- linux-headers-4.15.0-65-generic
- build-essential
override-build: |
# staging libraries conflict with other parts packages
unset CPPFLAGS
# make V=1 for verbose
make install --directory ${SNAPCRAFT_PART_BUILD} T=x86_64-native-linuxapp-gcc DESTDIR=$SNAPCRAFT_PART_INSTALL/dpdk RTE_KERNELDIR=/usr/src/linux-headers-4.15.0-65-generic # RTE_KERNELDIR is only needed if kernel version is not the same as installed on build host.

# Add Ubuntu Cloud Archive sources.
# Allows us to fetch things such as updated libvirt.
uca-sources:
@@ -502,7 +563,34 @@ parts:
prime:
- -*

openvswitch-dpdk:
# TODO: figure out a better way to fetch the version
source: http://openvswitch.org/releases/openvswitch-2.11.1.tar.gz
plugin: autotools
build-packages:
- libssl-dev
- try: [libnuma-dev]
- libcap-ng-dev
- libpcap-dev
- libunbound-dev
- python-all
- python-six
- python-setuptools
stage-packages:
- uuid-runtime
- libunbound2
configflags:
- "--localstatedir=/var/snap/$SNAPCRAFT_PROJECT_NAME/common"
- "--sysconfdir=/var/snap/$SNAPCRAFT_PROJECT_NAME/etc"
- "--with-dpdk=$SNAPCRAFT_STAGE/dpdk"
after:
- patches
- dpdk
organize:
./*: openvswitch-dpdk/

openvswitch:
# TODO: figure out a better way to fetch the version
source: http://openvswitch.org/releases/openvswitch-2.11.1.tar.gz
plugin: autotools
build-packages:
@@ -522,12 +610,53 @@ parts:
- "--sysconfdir=/var/snap/$SNAPCRAFT_PROJECT_NAME/etc"
after:
- patches
organize:
./*: openvswitch/

ovs-alternatives:
source: ./snap-wrappers/ovs
plugin: dump
after:
- openvswitch
- openvswitch-dpdk
organize:
ovs-alternatives: bin/ovs-alternatives

ovs-alternatives-init:
plugin: nil
after:
- openvswitch
- openvswitch-dpdk
override-prime: |
# assume we don't have different binaries for the two builds
# Create initial links for openvswitch binaries
for dir in \
bin \
etc/init.d \
sbin \
share/openvswitch/scripts \
usr/bin \
var/snap/${SNAPCRAFT_PROJECT_NAME}/etc/bash_completion.d \
; do
cd ${SNAPCRAFT_PRIME}/openvswitch/
mkdir -p ${SNAPCRAFT_PRIME}/${dir}/real
for file in ${dir}/*; do
ln -fs /snap/${SNAPCRAFT_PROJECT_NAME}/current/bin/ovs-wrapper ${SNAPCRAFT_PRIME}/${dir}/${file##*/}
ln -fs /var/snap/${SNAPCRAFT_PROJECT_NAME}/common/${dir}/${file##*/} ${SNAPCRAFT_PRIME}/${dir}/real/${file##*/}
done
cd -
done
# special file: ovs-lib
# there is no way in POSIX sh to detect if being sourced from another script
ln -fs /snap/${SNAPCRAFT_PROJECT_NAME}/current/share/openvswitch/scripts/real/ovs-lib ${SNAPCRAFT_PRIME}/share/openvswitch/scripts/ovs-lib
snapcraftctl prime

ovs-wrapper:
source: ./snap-wrappers/ovs
plugin: dump
after:
- openvswitch
- openvswitch-dpdk
organize:
ovs-wrapper: bin/ovs-wrapper

@@ -536,6 +665,8 @@ parts:
plugin: autotools
after:
- patches
- uca-sources
- dpdk
configflags:
- --prefix=/usr
- "--http-log-path=/var/snap/$SNAPCRAFT_PROJECT_NAME/common/log/nginx-access.log"
@@ -668,7 +799,7 @@ parts:
# required to ensure that pathing to files etc works at
# runtime
# * is not used to avoid directory merge conflicts
snap/microstack/current/: ./
snap/${SNAPCRAFT_PROJECT_NAME}/current/: ./

kvm-support:
plugin: nil
@@ -755,7 +886,7 @@ parts:
# required to ensure that pathing to files etc works at
# runtime
# * is not used to avoid directory merge conflicts
snap/microstack/current/: ./
snap/${SNAPCRAFT_PROJECT_NAME}/current/: ./

# MySQL
mysql-server:

+ 82
- 45
tools/init/init/questions/__init__.py View File

@@ -170,12 +170,20 @@ class NetworkSettings(Question):
"""Write network settings, and """
_type = 'auto'
_question = 'Network settings'
interactive = True

def yes(self, answer):
log.info('Configuring networking ...')

network.ExtGateway().ask()
network.ExtCidr().ask()
questions = [
network.ExtGateway(),
network.ExtCidr(),
network.OvsDpdk(),
]
for question in questions:
if not self.interactive:
question.interactive = False
question.ask()

# Now that we have default or overriden values, setup the
# bridge and write all the proper values into our config
@@ -183,7 +191,13 @@ class NetworkSettings(Question):
check('setup-br-ex')
check('snap-openstack', 'setup')

network.IpForwarding().ask()
questions = [
network.IpForwarding(),
]
for question in questions:
if not self.interactive:
question.interactive = False
question.ask()


class OsPassword(ConfigQuestion):
@@ -352,8 +366,9 @@ class DatabaseSetup(Question):

# Start keystone-uwsgi. We use snapctl, because systemd
# doesn't yet know about the service.
check('snapctl', 'start', 'microstack.nginx')
check('snapctl', 'start', 'microstack.keystone-uwsgi')
check('snapctl', 'start', '{SNAP_INSTANCE_NAME}.nginx'.format(**_env))
check('snapctl', 'start',
'{SNAP_INSTANCE_NAME}.keystone-uwsgi'.format(**_env))

log.info('Configuring Keystone Fernet Keys ...')
check('snap-openstack', 'launch', 'keystone-manage',
@@ -388,6 +403,12 @@ class NovaHypervisor(Question):
_type = 'boolean'
config_key = 'config.services.hypervisor'

@property
def services(self):
return [
'{SNAP_INSTANCE_NAME}.nova-compute'.format(**_env)
]

def yes(self, answer):
log.info('Configuring nova compute hypervisor ...')

@@ -401,11 +422,14 @@ class NovaHypervisor(Question):
'microstack', 'compute', endpoint,
'http://{compute_ip}:8774/v2.1'.format(**_env))

check('snapctl', 'start', 'microstack.nova-compute')
for service in self.services:
check('snapctl', 'start', service)

def no(self, answer):
log.info('Disabling nova compute service ...')
check('systemctl', 'disable', 'snap.microstack.nova-compute')

for service in self.services:
check('systemctl', 'disable', 'snap.{}'.format(service))


class NovaControlPlane(Question):
@@ -414,6 +438,17 @@ class NovaControlPlane(Question):
_type = 'boolean'
config_key = 'config.services.control-plane'

@property
def services(self):
return [
'{SNAP_INSTANCE_NAME}.nova-api'.format(**_env),
'{SNAP_INSTANCE_NAME}.nova-api-metadata'.format(**_env),
'{SNAP_INSTANCE_NAME}.nova-compute'.format(**_env),
'{SNAP_INSTANCE_NAME}.nova-conductor'.format(**_env),
'{SNAP_INSTANCE_NAME}.nova-scheduler'.format(**_env),
'{SNAP_INSTANCE_NAME}.nova-uwsgi'.format(**_env)
]

def _flavors(self) -> None:
"""Create default flavors."""

@@ -463,13 +498,7 @@ class NovaControlPlane(Question):
# out manually, because systemd doesn't know about them yet.
# TODO: parse the output of `snapctl services` to get this
# list automagically.
for service in [
'microstack.nova-api',
'microstack.nova-api-metadata',
'microstack.nova-conductor',
'microstack.nova-scheduler',
'microstack.nova-uwsgi',
]:
for service in self.services:
check('snapctl', 'start', service)

check('snap-openstack', 'launch', 'nova-manage', 'api_db', 'sync')
@@ -500,14 +529,10 @@ class NovaControlPlane(Question):
def no(self, answer):
log.info('Disabling nova control plane services ...')

for service in [
'snap.microstack.nova-uwsgi',
'snap.microstack.nova-api',
'snap.microstack.nova-conductor',
'snap.microstack.nova-scheduler',
'snap.microstack.nova-api-metadata']:
nova_compute = '{SNAP_INSTANCE_NAME}.nova-compute'.format(**_env)

check('systemctl', 'disable', service)
for service in self.services.remove(nova_compute) or []:
check('systemctl', 'disable', 'snap.{}'.format(service))


class NeutronControlPlane(Question):
@@ -516,6 +541,16 @@ class NeutronControlPlane(Question):
_type = 'boolean'
config_key = 'config.services.control-plane'

@property
def services(self):
return [
'{SNAP_INSTANCE_NAME}.neutron-api'.format(**_env),
'{SNAP_INSTANCE_NAME}.neutron-dhcp-agent'.format(**_env),
'{SNAP_INSTANCE_NAME}.neutron-l3-agent'.format(**_env),
'{SNAP_INSTANCE_NAME}.neutron-metadata-agent'.format(**_env),
'{SNAP_INSTANCE_NAME}.neutron-openvswitch-agent'.format(**_env),
]

def yes(self, answer: str) -> None:
log.info('Configuring Neutron')

@@ -533,13 +568,7 @@ class NeutronControlPlane(Question):
'microstack', 'network', endpoint,
'http://{control_ip}:9696'.format(**_env))

for service in [
'microstack.neutron-api',
'microstack.neutron-dhcp-agent',
'microstack.neutron-l3-agent',
'microstack.neutron-metadata-agent',
'microstack.neutron-openvswitch-agent',
]:
for service in self.services:
check('snapctl', 'start', service)

check('snap-openstack', 'launch', 'neutron-db-manage', 'upgrade',
@@ -579,20 +608,16 @@ class NeutronControlPlane(Question):
neutron on this machine.

"""
openvswitch_agent = """snap.{SNAP_INSTANCE_NAME}.\
neutron-openvswitch-agent""".format(**_env)

# Make sure that the agent is running.
for service in [
'microstack.neutron-openvswitch-agent',
]:
for service in [openvswitch_agent]:
check('snapctl', 'start', service)

# Disable the other services.
for service in [
'snap.microstack.neutron-api',
'snap.microstack.neutron-dhcp-agent',
'snap.microstack.neutron-metadata-agent',
'snap.microstack.neutron-l3-agent',
]:
check('systemctl', 'disable', service)
for service in self.services.remove(openvswitch_agent) or []:
check('systemctl', 'disable', 'snap.{}'.format(service))


class GlanceSetup(Question):
@@ -601,6 +626,14 @@ class GlanceSetup(Question):
_type = 'boolean'
config_key = 'config.services.control-plane'

@property
def services(self):
return [
'{SNAP_INSTANCE_NAME}.glance-api'.format(**_env),
# TODO rename this to glance-registry
'{SNAP_INSTANCE_NAME}.registry'.format(**_env),
]

def _fetch_cirros(self) -> None:

if call('openstack', 'image', 'show', 'cirros'):
@@ -644,10 +677,7 @@ class GlanceSetup(Question):
'microstack', 'image', endpoint,
'http://{compute_ip}:9292'.format(**_env))

for service in [
'microstack.glance-api',
'microstack.registry', # TODO rename to glance-registery
]:
for service in self.services:
check('snapctl', 'start', service)

check('snap-openstack', 'launch', 'glance-manage', 'db_sync')
@@ -661,8 +691,8 @@ class GlanceSetup(Question):
self._fetch_cirros()

def no(self, answer):
check('systemctl', 'disable', 'snap.microstack.glance-api')
check('systemctl', 'disable', 'snap.microstack.registry')
for service in self.services:
check('systemctl', 'disable', 'snap.{}'.format(service))


class KeyPair(Question):
@@ -731,6 +761,12 @@ class PostSetup(Question):

config_key = 'config.post-setup'

@property
def services(self):
return [
'{SNAP_INSTANCE_NAME}.horizon-uwsgi'.format(**_env)
]

def yes(self, answer: str) -> None:

log.info('restarting libvirt and virtlogd ...')
@@ -739,7 +775,8 @@ class PostSetup(Question):
restart('*virt*')

# Start horizon
check('snapctl', 'start', 'microstack.horizon-uwsgi')
for service in self.services:
check('snapctl', 'start', service)

check('snapctl', 'set', 'initialized=true')
log.info('Complete. Marked microstack as initialized!')

+ 61
- 1
tools/init/init/questions/network.py View File

@@ -1,6 +1,7 @@
from init.config import Env, log
from init.questions.question import Question
from init.shell import check, check_output
from init.shell import check, check_output, restart
from os import path, remove

_env = Env().get_env()

@@ -50,3 +51,62 @@ class IpForwarding(Question):
log.info('Setting up ipv4 forwarding...')

check('sysctl', 'net.ipv4.ip_forward=1')


class OvsDpdk(Question):
"""Possibly setup OVS DPDK."""

_type = 'boolean'
_question = 'Do you wish to setup OVS DPDK?'
config_key = 'config.network.ovs-dpdk'
interactive = True

def yes(self, answer: bool):
"""Use ovs-vsctl to setup ovs-dpdk"""
log.info('Setting up OVS DPDK...')
check('snapctl', 'set', 'config.network.ovs-dpdk={}'.format(answer))

check('ovs-wrapper', 'ovs-vsctl', '--no-wait', 'set', 'Open_vSwitch',
'.', 'other_config:dpdk-init=true',
'other_config:dpdk-socket-mem=1024,0',
'other_config:pmd-cpu-mask=0x3')

_path = """{SNAP_COMMON}/etc/neutron/neutron.conf.d/\
neutron-dpdk.conf""".format(**_env)

with open(_path, 'w') as _file:
_file.write("""\
[OVS]
datapath_type = netdev
vhostuser_socket_dir = {SNAP_COMMON}/run/openvswitch
""".format(**_env))

# (re)configure alternatives based on the dpdk answer
check('ovs-alternatives', '--install')

restart('ovs*')
restart('neutron*')

def no(self, answer: bool):
log.info('Setting up OVS...')
check('snapctl', 'set', 'config.network.ovs-dpdk={}'.format(answer))

# only remove config kept on default
check('ovs-wrapper', 'ovs-vsctl', '--no-wait', 'remove',
'Open_vSwitch', '.',
'other_config', 'dpdk-init', 'true',
'other_config', 'pmd-cpu-mask', '0x3',
'dpdk-socket-mem', '1024,0')

_path = """{SNAP_COMMON}/etc/neutron/neutron.conf.d/\
neutron-dpdk.conf""".format(**_env)

if path.exists(_path):
remove(_path)

# (re)configure alternatives based on the dpdk answer
check('ovs-alternatives', '--remove')
check('ovs-alternatives', '--install')

restart('ovs*')
restart('neutron*')

+ 5
- 2
tools/init/init/shell.py View File

@@ -35,7 +35,6 @@ import wget

from init.config import Env, log


_env = Env().get_env()


@@ -159,7 +158,11 @@ def restart(service: str) -> None:
e.g. *rabbit*

"""
check('systemctl', 'restart', 'snap.microstack.{}'.format(service))
check(
'systemctl',
'restart',
'snap.{SNAP_INSTANCE_NAME}.{SERVICE}'.format(**_env, SERVICE=service)
)


def disable(service: str) -> None:

Loading…
Cancel
Save