x/microstack
Code Issues Proposed changes

Initial DPDK support

Browse Source 
Adding initial support for OVS DPDK.

Signed-off-by: Gabor Meszaros <gabor@meszaros.pw>
Change-Id: I1f87188bd8b4b928108674a35e9a9c309e9f6d17
This commit is contained in:
Gabor Meszaros 2019-12-04 10:54:52 +01:00
parent 1a25e50a17
commit ce5e82e319
11 changed files with 398 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
snap-overlay/bin/set-default-config
View File

@ -23,6 +23,7 @@ snapctl set \
config.network.ports.dashboard=80 \
config.network.ports.mysql=3306 \
config.network.ports.rabbit=5672 \
config.network.ovs-dpdk=false \
;
# Passwords, certs, etc.

8
snap-overlay/bin/setup-br-ex
View File

@ -12,13 +12,15 @@ set -ex
extcidr=$(snapctl get config.network.ext-cidr)
# Create external integration bridge
ovs-vsctl --retry --may-exist add-br br-ex
ovs-wrapper ovs-vsctl --retry --may-exist add-br br-ex
[ $(snapctl get config.network.ovs-dpdk) == true ] && \
ovs-wrapper ovs-vsctl -- set Bridge br-ex datapath_type=netdev
# Configure br-ex
ip address add $extcidr dev br-ex || :
ip link set br-ex up || :
sudo iptables -w -t nat -A POSTROUTING -s $extcidr ! \
-d $extcidr -j MASQUERADE || :
sudo iptables -w -t nat -C POSTROUTING -s $extcidr ! -d $extcidr -j MASQUERADE || \
sudo iptables -w -t nat -A POSTROUTING -s $extcidr ! -d $extcidr -j MASQUERADE
exit 0

70
snap-wrappers/ovs/ovs-alternatives Executable file
View File

@ -0,0 +1,70 @@
#!/bin/bash
# this initializes the ovs commands alternatives with the default non-dpdk binaries
set -e
function _remove(){
#remove both openvswitch and openvswitch-dpdk alternatives of the current release.
# it is called at post-refresh hook time and at remove hook time also.
for version in openvswitch openvswitch-dpdk; do
for dir in \
bin \
etc/init.d \
sbin \
share/openvswitch/scripts \
usr/bin \
var/snap/${SNAP_NAME}/etc/bash_completion.d \
; do
cd ${SNAP}/${version}/${dir}/
for file in *; do
update-alternatives --quiet --remove ${file} ${SNAP}/${version}/${dir}/${file}
done
cd - &>/dev/null
done
done
}
function _install(){
# only set version alternatives when it is configured via snap config options
version="openvswitch"
if [[ $(snapctl get config.network.ovs-dpdk) == *"rue" ]]; then
version="openvswitch-dpdk"
fi
for dir in \
bin \
etc/init.d \
sbin \
share/openvswitch/scripts \
usr/bin \
var/snap/${SNAP_NAME}/etc/bash_completion.d \
; do
cd ${SNAP}/${version}/${dir}/
mkdir -p ${SNAP_COMMON}/${dir}/
for file in *; do
update-alternatives --quiet --install $SNAP_COMMON/$dir/${file} ${file} ${SNAP}/${version}/${dir}/${file} 1
update-alternatives --quiet --set ${file} ${SNAP}/${version}/${dir}/${file}
done
cd - &>/dev/null
done
}
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
-i|--install)
shift # past argument
_install
;;
-r|--remove)
shift # past argument
_remove
;;
*) # unknown option, noop
shift # past argument
;;
esac
done
exit 0

34
snap-wrappers/ovs/ovs-wrapper
View File

@ -1,16 +1,42 @@
#!/bin/bash
# important: not to echo or print anything on stdout, will be taken into ovs-ctl
set -e
export OVS_LOGDIR=${SNAP_COMMON}/log/openvswitch
export OVS_RUNDIR=${SNAP_COMMON}/run/openvswitch
export OVS_SYSCONFDIR=${SNAP_COMMON}/etc
export OVS_PKGDATADIR=${SNAP}/share/openvswitch
export OVS_BINDIR=${SNAP}/bin
export OVS_SBINDIR=${SNAP}/sbin
export OVS_PKGDATADIR=${SNAP}/openvswitch-dpdk/share/openvswitch
if [ "$(snapctl get config.network.ovs-dpdk)" == "false" ]; then
export OVS_BINDIR=${SNAP}/openvswitch/bin
export OVS_SBINDIR=${SNAP}/openvswitch/sbin
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${SNAP}/openvswitch/lib:${SNAP}/openvswitch/usr/lib/${SNAPCRAFT_ARCH_TRIPLET}"
else
export OVS_BINDIR=${SNAP}/openvswitch-dpdk/bin
export OVS_SBINDIR=${SNAP}/openvswitch-dpdk/sbin
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${SNAP}/openvswitch-dpdk/lib:${SNAP}/openvswitch-dpdk/usr/lib/${SNAPCRAFT_ARCH_TRIPLET}"
fi
mkdir -p ${OVS_LOGDIR}
mkdir -p ${OVS_RUNDIR}
mkdir -p ${OVS_SYSCONFDIR}/openvswitch
exec $@
# need to maintain ovsdb-server and ovs-vswtichd unix socket path:
shopt -s nullglob
for ctl in $OVS_RUNDIR/*.real.*.ctl;
do
ln -fs $ctl "${ctl//.real/}"
done
# needs further testing: vvv
#echo "find $OVS_RUNDIR/*.ctl -xtype l -delete"
shopt -u nullglob
if [[ "$0" == *"ovs-wrapper"* ]]; then
cmd=$(command -v ${1})
exec ${cmd%/*}/real/${cmd##*/} "${@:2}"
else
# command is symlink to this script
exec "${0%/*}/real/${0##*/}" "${@}"
fi

3
snap/hooks/configure vendored
View File

@ -1,6 +1,9 @@
#!/bin/bash
set -ex
# update the openvswitch binaries alternatives if needed
${SNAP}/bin/ovs-alternatives --install
snap-openstack setup # Write out templates
source $SNAP_COMMON/etc/microstack.rc

3
snap/hooks/install
View File

@ -22,4 +22,7 @@ done
# Make a place for our horizon config overrides to live
mkdir -p ${SNAP_COMMON}/etc/horizon/local_settings.d
# provide initial openvswitch binaries alternatives
${SNAP}/bin/ovs-alternatives --install
snap-openstack setup # Sets up templates for the first time.

5
snap/hooks/remove Executable file
View File

@ -0,0 +1,5 @@
#!/bin/bash
set -ex
# cleanup openvswitch binaries alternatives
${SNAP}/bin/ovs-alternatives --remove

135
snapcraft.yaml
View File

@ -318,8 +318,69 @@ apps:
join:
command: python3 ${SNAP}/lib/python3.6/site-packages/cluster/client.py
testpmd:
command: dpdk/bin/testpmd
#plugs:
# - network
# - network-bind
# - network-control
# - network-observe
# - hardware-observe
# #- hugepages-observe
# #- hugepages-control
# - process-control
# - system-observe
# #- dpdk-control
# - kernel-module-control
# - kernel-module-observe
dpdk-devbind:
command: dpdk/sbin/dpdk-devbind
#plugs:
# - network
# - network-bind
# - network-control
# - network-observe
# - hardware-observe
# - system-observe
# #- dpdk-control
# - kernel-module-control
# - kernel-module-observe
dpdk-setup:
command: dpdk/share/dpdk/usertools/dpdk-setup.sh
#plugs:
# - network
# - network-bind
# - network-control
# - network-observe
# - hardware-observe
# #- system-observer
# #- dpdk-control
# - kernel-module-control
# - kernel-module-observe
ovs-alternatives:
command: bin/ovs-alternatives
parts:
dpdk:
plugin: make
source: http://dpdk.org/git/dpdk-stable
source-type: git
source-tag: v18.11 # LTS Version, comment out or change tag to what version you require.
after:
- patches
- uca-sources
build-packages:
- linux-headers-4.15.0-65-generic
- build-essential
override-build: |
# staging libraries conflict with other parts packages
unset CPPFLAGS
# make V=1 for verbose
make install --directory ${SNAPCRAFT_PART_BUILD} T=x86_64-native-linuxapp-gcc DESTDIR=$SNAPCRAFT_PART_INSTALL/dpdk RTE_KERNELDIR=/usr/src/linux-headers-4.15.0-65-generic # RTE_KERNELDIR is only needed if kernel version is not the same as installed on build host.
# Add Ubuntu Cloud Archive sources.
# Allows us to fetch things such as updated libvirt.
uca-sources:
@ -502,7 +563,34 @@ parts:
prime:
- -*
openvswitch-dpdk:
# TODO: figure out a better way to fetch the version
source: http://openvswitch.org/releases/openvswitch-2.11.1.tar.gz
plugin: autotools
build-packages:
- libssl-dev
- try: [libnuma-dev]
- libcap-ng-dev
- libpcap-dev
- libunbound-dev
- python-all
- python-six
- python-setuptools
stage-packages:
- uuid-runtime
- libunbound2
configflags:
- "--localstatedir=/var/snap/$SNAPCRAFT_PROJECT_NAME/common"
- "--sysconfdir=/var/snap/$SNAPCRAFT_PROJECT_NAME/etc"
- "--with-dpdk=$SNAPCRAFT_STAGE/dpdk"
after:
- patches
- dpdk
organize:
./*: openvswitch-dpdk/
openvswitch:
# TODO: figure out a better way to fetch the version
source: http://openvswitch.org/releases/openvswitch-2.11.1.tar.gz
plugin: autotools
build-packages:
@ -522,12 +610,53 @@ parts:
- "--sysconfdir=/var/snap/$SNAPCRAFT_PROJECT_NAME/etc"
after:
- patches
organize:
./*: openvswitch/
ovs-alternatives:
source: ./snap-wrappers/ovs
plugin: dump
after:
- openvswitch
- openvswitch-dpdk
organize:
ovs-alternatives: bin/ovs-alternatives
ovs-alternatives-init:
plugin: nil
after:
- openvswitch
- openvswitch-dpdk
override-prime: |
# assume we don't have different binaries for the two builds
# Create initial links for openvswitch binaries
for dir in \
bin \
etc/init.d \
sbin \
share/openvswitch/scripts \
usr/bin \
var/snap/${SNAPCRAFT_PROJECT_NAME}/etc/bash_completion.d \
; do
cd ${SNAPCRAFT_PRIME}/openvswitch/
mkdir -p ${SNAPCRAFT_PRIME}/${dir}/real
for file in ${dir}/*; do
ln -fs /snap/${SNAPCRAFT_PROJECT_NAME}/current/bin/ovs-wrapper ${SNAPCRAFT_PRIME}/${dir}/${file##*/}
ln -fs /var/snap/${SNAPCRAFT_PROJECT_NAME}/common/${dir}/${file##*/} ${SNAPCRAFT_PRIME}/${dir}/real/${file##*/}
done
cd -
done
# special file: ovs-lib
# there is no way in POSIX sh to detect if being sourced from another script
ln -fs /snap/${SNAPCRAFT_PROJECT_NAME}/current/share/openvswitch/scripts/real/ovs-lib ${SNAPCRAFT_PRIME}/share/openvswitch/scripts/ovs-lib
snapcraftctl prime
ovs-wrapper:
source: ./snap-wrappers/ovs
plugin: dump
after:
- openvswitch
- openvswitch-dpdk
organize:
ovs-wrapper: bin/ovs-wrapper
@ -536,6 +665,8 @@ parts:
plugin: autotools
after:
- patches
- uca-sources
- dpdk
configflags:
- --prefix=/usr
- "--http-log-path=/var/snap/$SNAPCRAFT_PROJECT_NAME/common/log/nginx-access.log"
@ -668,7 +799,7 @@ parts:
# required to ensure that pathing to files etc works at
# runtime
# * is not used to avoid directory merge conflicts
snap/microstack/current/: ./
snap/${SNAPCRAFT_PROJECT_NAME}/current/: ./
kvm-support:
plugin: nil
@ -755,7 +886,7 @@ parts:
# required to ensure that pathing to files etc works at
# runtime
# * is not used to avoid directory merge conflicts
snap/microstack/current/: ./
snap/${SNAPCRAFT_PROJECT_NAME}/current/: ./
# MySQL
mysql-server:

127
tools/init/init/questions/__init__.py
View File

@ -170,12 +170,20 @@ class NetworkSettings(Question):
"""Write network settings, and """
_type = 'auto'
_question = 'Network settings'
interactive = True
def yes(self, answer):
log.info('Configuring networking ...')
network.ExtGateway().ask()
network.ExtCidr().ask()
questions = [
network.ExtGateway(),
network.ExtCidr(),
network.OvsDpdk(),
]
for question in questions:
if not self.interactive:
question.interactive = False
question.ask()
# Now that we have default or overriden values, setup the
# bridge and write all the proper values into our config
@ -183,7 +191,13 @@ class NetworkSettings(Question):
check('setup-br-ex')
check('snap-openstack', 'setup')
network.IpForwarding().ask()
questions = [
network.IpForwarding(),
]
for question in questions:
if not self.interactive:
question.interactive = False
question.ask()
class OsPassword(ConfigQuestion):
@ -352,8 +366,9 @@ class DatabaseSetup(Question):
# Start keystone-uwsgi. We use snapctl, because systemd
# doesn't yet know about the service.
check('snapctl', 'start', 'microstack.nginx')
check('snapctl', 'start', 'microstack.keystone-uwsgi')
check('snapctl', 'start', '{SNAP_INSTANCE_NAME}.nginx'.format(**_env))
check('snapctl', 'start',
'{SNAP_INSTANCE_NAME}.keystone-uwsgi'.format(**_env))
log.info('Configuring Keystone Fernet Keys ...')
check('snap-openstack', 'launch', 'keystone-manage',
@ -388,6 +403,12 @@ class NovaHypervisor(Question):
_type = 'boolean'
config_key = 'config.services.hypervisor'
@property
def services(self):
return [
'{SNAP_INSTANCE_NAME}.nova-compute'.format(**_env)
]
def yes(self, answer):
log.info('Configuring nova compute hypervisor ...')
@ -401,11 +422,14 @@ class NovaHypervisor(Question):
'microstack', 'compute', endpoint,
'http://{compute_ip}:8774/v2.1'.format(**_env))
check('snapctl', 'start', 'microstack.nova-compute')
for service in self.services:
check('snapctl', 'start', service)
def no(self, answer):
log.info('Disabling nova compute service ...')
check('systemctl', 'disable', 'snap.microstack.nova-compute')
for service in self.services:
check('systemctl', 'disable', 'snap.{}'.format(service))
class NovaControlPlane(Question):
@ -414,6 +438,17 @@ class NovaControlPlane(Question):
_type = 'boolean'
config_key = 'config.services.control-plane'
@property
def services(self):
return [
'{SNAP_INSTANCE_NAME}.nova-api'.format(**_env),
'{SNAP_INSTANCE_NAME}.nova-api-metadata'.format(**_env),
'{SNAP_INSTANCE_NAME}.nova-compute'.format(**_env),
'{SNAP_INSTANCE_NAME}.nova-conductor'.format(**_env),
'{SNAP_INSTANCE_NAME}.nova-scheduler'.format(**_env),
'{SNAP_INSTANCE_NAME}.nova-uwsgi'.format(**_env)
]
def _flavors(self) -> None:
"""Create default flavors."""
@ -463,13 +498,7 @@ class NovaControlPlane(Question):
# out manually, because systemd doesn't know about them yet.
# TODO: parse the output of `snapctl services` to get this
# list automagically.
for service in [
'microstack.nova-api',
'microstack.nova-api-metadata',
'microstack.nova-conductor',
'microstack.nova-scheduler',
'microstack.nova-uwsgi',
]:
for service in self.services:
check('snapctl', 'start', service)
check('snap-openstack', 'launch', 'nova-manage', 'api_db', 'sync')
@ -500,14 +529,10 @@ class NovaControlPlane(Question):
def no(self, answer):
log.info('Disabling nova control plane services ...')
for service in [
'snap.microstack.nova-uwsgi',
'snap.microstack.nova-api',
'snap.microstack.nova-conductor',
'snap.microstack.nova-scheduler',
'snap.microstack.nova-api-metadata']:
nova_compute = '{SNAP_INSTANCE_NAME}.nova-compute'.format(**_env)
check('systemctl', 'disable', service)
for service in self.services.remove(nova_compute) or []:
check('systemctl', 'disable', 'snap.{}'.format(service))
class NeutronControlPlane(Question):
@ -516,6 +541,16 @@ class NeutronControlPlane(Question):
_type = 'boolean'
config_key = 'config.services.control-plane'
@property
def services(self):
return [
'{SNAP_INSTANCE_NAME}.neutron-api'.format(**_env),
'{SNAP_INSTANCE_NAME}.neutron-dhcp-agent'.format(**_env),
'{SNAP_INSTANCE_NAME}.neutron-l3-agent'.format(**_env),
'{SNAP_INSTANCE_NAME}.neutron-metadata-agent'.format(**_env),
'{SNAP_INSTANCE_NAME}.neutron-openvswitch-agent'.format(**_env),
]
def yes(self, answer: str) -> None:
log.info('Configuring Neutron')
@ -533,13 +568,7 @@ class NeutronControlPlane(Question):
'microstack', 'network', endpoint,
'http://{control_ip}:9696'.format(**_env))
for service in [
'microstack.neutron-api',
'microstack.neutron-dhcp-agent',
'microstack.neutron-l3-agent',
'microstack.neutron-metadata-agent',
'microstack.neutron-openvswitch-agent',
]:
for service in self.services:
check('snapctl', 'start', service)
check('snap-openstack', 'launch', 'neutron-db-manage', 'upgrade',
@ -579,20 +608,16 @@ class NeutronControlPlane(Question):
neutron on this machine.
"""
openvswitch_agent = """snap.{SNAP_INSTANCE_NAME}.\
neutron-openvswitch-agent""".format(**_env)
# Make sure that the agent is running.
for service in [
'microstack.neutron-openvswitch-agent',
]:
for service in [openvswitch_agent]:
check('snapctl', 'start', service)
# Disable the other services.
for service in [
'snap.microstack.neutron-api',
'snap.microstack.neutron-dhcp-agent',
'snap.microstack.neutron-metadata-agent',
'snap.microstack.neutron-l3-agent',
]:
check('systemctl', 'disable', service)
for service in self.services.remove(openvswitch_agent) or []:
check('systemctl', 'disable', 'snap.{}'.format(service))
class GlanceSetup(Question):
@ -601,6 +626,14 @@ class GlanceSetup(Question):
_type = 'boolean'
config_key = 'config.services.control-plane'
@property
def services(self):
return [
'{SNAP_INSTANCE_NAME}.glance-api'.format(**_env),
# TODO rename this to glance-registry
'{SNAP_INSTANCE_NAME}.registry'.format(**_env),
]
def _fetch_cirros(self) -> None:
if call('openstack', 'image', 'show', 'cirros'):
@ -644,10 +677,7 @@ class GlanceSetup(Question):
'microstack', 'image', endpoint,
'http://{compute_ip}:9292'.format(**_env))
for service in [
'microstack.glance-api',
'microstack.registry', # TODO rename to glance-registery
]:
for service in self.services:
check('snapctl', 'start', service)
check('snap-openstack', 'launch', 'glance-manage', 'db_sync')
@ -661,8 +691,8 @@ class GlanceSetup(Question):
self._fetch_cirros()
def no(self, answer):
check('systemctl', 'disable', 'snap.microstack.glance-api')
check('systemctl', 'disable', 'snap.microstack.registry')
for service in self.services:
check('systemctl', 'disable', 'snap.{}'.format(service))
class KeyPair(Question):
@ -731,6 +761,12 @@ class PostSetup(Question):
config_key = 'config.post-setup'
@property
def services(self):
return [
'{SNAP_INSTANCE_NAME}.horizon-uwsgi'.format(**_env)
]
def yes(self, answer: str) -> None:
log.info('restarting libvirt and virtlogd ...')
@ -739,7 +775,8 @@ class PostSetup(Question):
restart('*virt*')
# Start horizon
check('snapctl', 'start', 'microstack.horizon-uwsgi')
for service in self.services:
check('snapctl', 'start', service)
check('snapctl', 'set', 'initialized=true')
log.info('Complete. Marked microstack as initialized!')

62
tools/init/init/questions/network.py
View File

@ -1,6 +1,7 @@
from init.config import Env, log
from init.questions.question import Question
from init.shell import check, check_output
from init.shell import check, check_output, restart
from os import path, remove
_env = Env().get_env()
@ -50,3 +51,62 @@ class IpForwarding(Question):
log.info('Setting up ipv4 forwarding...')
check('sysctl', 'net.ipv4.ip_forward=1')
class OvsDpdk(Question):
"""Possibly setup OVS DPDK."""
_type = 'boolean'
_question = 'Do you wish to setup OVS DPDK?'
config_key = 'config.network.ovs-dpdk'
interactive = True
def yes(self, answer: bool):
"""Use ovs-vsctl to setup ovs-dpdk"""
log.info('Setting up OVS DPDK...')
check('snapctl', 'set', 'config.network.ovs-dpdk={}'.format(answer))
check('ovs-wrapper', 'ovs-vsctl', '--no-wait', 'set', 'Open_vSwitch',
'.', 'other_config:dpdk-init=true',
'other_config:dpdk-socket-mem=1024,0',
'other_config:pmd-cpu-mask=0x3')
_path = """{SNAP_COMMON}/etc/neutron/neutron.conf.d/\
neutron-dpdk.conf""".format(**_env)
with open(_path, 'w') as _file:
_file.write("""\
[OVS]
datapath_type = netdev
vhostuser_socket_dir = {SNAP_COMMON}/run/openvswitch
""".format(**_env))
# (re)configure alternatives based on the dpdk answer
check('ovs-alternatives', '--install')
restart('ovs*')
restart('neutron*')
def no(self, answer: bool):
log.info('Setting up OVS...')
check('snapctl', 'set', 'config.network.ovs-dpdk={}'.format(answer))
# only remove config kept on default
check('ovs-wrapper', 'ovs-vsctl', '--no-wait', 'remove',
'Open_vSwitch', '.',
'other_config', 'dpdk-init', 'true',
'other_config', 'pmd-cpu-mask', '0x3',
'dpdk-socket-mem', '1024,0')
_path = """{SNAP_COMMON}/etc/neutron/neutron.conf.d/\
neutron-dpdk.conf""".format(**_env)
if path.exists(_path):
remove(_path)
# (re)configure alternatives based on the dpdk answer
check('ovs-alternatives', '--remove')
check('ovs-alternatives', '--install')
restart('ovs*')
restart('neutron*')

7
tools/init/init/shell.py
View File

@ -35,7 +35,6 @@ import wget
from init.config import Env, log
_env = Env().get_env()
@ -159,7 +158,11 @@ def restart(service: str) -> None:
e.g. *rabbit*
"""
check('systemctl', 'restart', 'snap.microstack.{}'.format(service))
check(
'systemctl',
'restart',
'snap.{SNAP_INSTANCE_NAME}.{SERVICE}'.format(**_env, SERVICE=service)
)
def disable(service: str) -> None: