x/microstack
Code Issues Proposed changes

Revert "Initial DPDK support"

Browse Source 
This reverts commit ce5e82e319.

MicroStack cannot currently install due to a missing ovs-related
library. This is possibly due to recent changes in spacraft, or
possibly due to the workarounds for those changes. Regardless, it
appears that backing out the DPDK changes gets us back to a state
where we can install.

Partial-Bug: 1862911
Change-Id: I060c1a0095470639f9158cb9e9ebe8281a65a678
This commit is contained in:
Pete Vander Giessen 2020-02-12 18:11:02 +00:00
parent 9019ad3529
commit d262cbed49
11 changed files with 57 additions and 398 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
snap-overlay/bin/set-default-config
View File

@ -23,7 +23,6 @@ snapctl set \
config.network.ports.dashboard=80 \
config.network.ports.mysql=3306 \
config.network.ports.rabbit=5672 \
config.network.ovs-dpdk=false \
;
# Passwords, certs, etc.

8
snap-overlay/bin/setup-br-ex
View File

@ -12,15 +12,13 @@ set -ex
extcidr=$(snapctl get config.network.ext-cidr)
# Create external integration bridge
ovs-wrapper ovs-vsctl --retry --may-exist add-br br-ex
[ $(snapctl get config.network.ovs-dpdk) == true ] && \
ovs-wrapper ovs-vsctl -- set Bridge br-ex datapath_type=netdev
ovs-vsctl --retry --may-exist add-br br-ex
# Configure br-ex
ip address add $extcidr dev br-ex || :
ip link set br-ex up || :
sudo iptables -w -t nat -C POSTROUTING -s $extcidr ! -d $extcidr -j MASQUERADE || \
sudo iptables -w -t nat -A POSTROUTING -s $extcidr ! -d $extcidr -j MASQUERADE
sudo iptables -w -t nat -A POSTROUTING -s $extcidr ! \
-d $extcidr -j MASQUERADE || :
exit 0

70
snap-wrappers/ovs/ovs-alternatives
View File

@ -1,70 +0,0 @@
#!/bin/bash
# this initializes the ovs commands alternatives with the default non-dpdk binaries
set -e
function _remove(){
#remove both openvswitch and openvswitch-dpdk alternatives of the current release.
# it is called at post-refresh hook time and at remove hook time also.
for version in openvswitch openvswitch-dpdk; do
for dir in \
bin \
etc/init.d \
sbin \
share/openvswitch/scripts \
usr/bin \
var/snap/${SNAP_NAME}/etc/bash_completion.d \
; do
cd ${SNAP}/${version}/${dir}/
for file in *; do
update-alternatives --quiet --remove ${file} ${SNAP}/${version}/${dir}/${file}
done
cd - &>/dev/null
done
done
}
function _install(){
# only set version alternatives when it is configured via snap config options
version="openvswitch"
if [[ $(snapctl get config.network.ovs-dpdk) == *"rue" ]]; then
version="openvswitch-dpdk"
fi
for dir in \
bin \
etc/init.d \
sbin \
share/openvswitch/scripts \
usr/bin \
var/snap/${SNAP_NAME}/etc/bash_completion.d \
; do
cd ${SNAP}/${version}/${dir}/
mkdir -p ${SNAP_COMMON}/${dir}/
for file in *; do
update-alternatives --quiet --install $SNAP_COMMON/$dir/${file} ${file} ${SNAP}/${version}/${dir}/${file} 1
update-alternatives --quiet --set ${file} ${SNAP}/${version}/${dir}/${file}
done
cd - &>/dev/null
done
}
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
-i|--install)
shift # past argument
_install
;;
-r|--remove)
shift # past argument
_remove
;;
*) # unknown option, noop
shift # past argument
;;
esac
done
exit 0

34
snap-wrappers/ovs/ovs-wrapper
View File

@ -1,42 +1,16 @@
#!/bin/bash
# important: not to echo or print anything on stdout, will be taken into ovs-ctl
set -e
export OVS_LOGDIR=${SNAP_COMMON}/log/openvswitch
export OVS_RUNDIR=${SNAP_COMMON}/run/openvswitch
export OVS_SYSCONFDIR=${SNAP_COMMON}/etc
export OVS_PKGDATADIR=${SNAP}/openvswitch-dpdk/share/openvswitch
if [ "$(snapctl get config.network.ovs-dpdk)" == "false" ]; then
export OVS_BINDIR=${SNAP}/openvswitch/bin
export OVS_SBINDIR=${SNAP}/openvswitch/sbin
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${SNAP}/openvswitch/lib:${SNAP}/openvswitch/usr/lib/${SNAPCRAFT_ARCH_TRIPLET}"
else
export OVS_BINDIR=${SNAP}/openvswitch-dpdk/bin
export OVS_SBINDIR=${SNAP}/openvswitch-dpdk/sbin
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${SNAP}/openvswitch-dpdk/lib:${SNAP}/openvswitch-dpdk/usr/lib/${SNAPCRAFT_ARCH_TRIPLET}"
fi
export OVS_PKGDATADIR=${SNAP}/share/openvswitch
export OVS_BINDIR=${SNAP}/bin
export OVS_SBINDIR=${SNAP}/sbin
mkdir -p ${OVS_LOGDIR}
mkdir -p ${OVS_RUNDIR}
mkdir -p ${OVS_SYSCONFDIR}/openvswitch
# need to maintain ovsdb-server and ovs-vswtichd unix socket path:
shopt -s nullglob
for ctl in $OVS_RUNDIR/*.real.*.ctl;
do
ln -fs $ctl "${ctl//.real/}"
done
# needs further testing: vvv
#echo "find $OVS_RUNDIR/*.ctl -xtype l -delete"
shopt -u nullglob
if [[ "$0" == *"ovs-wrapper"* ]]; then
cmd=$(command -v ${1})
exec ${cmd%/*}/real/${cmd##*/} "${@:2}"
else
# command is symlink to this script
exec "${0%/*}/real/${0##*/}" "${@}"
fi
exec $@

3
snap/hooks/configure vendored
View File

@ -1,9 +1,6 @@
#!/bin/bash
set -ex
# update the openvswitch binaries alternatives if needed
${SNAP}/bin/ovs-alternatives --install
snap-openstack setup # Write out templates
source $SNAP_COMMON/etc/microstack.rc

3
snap/hooks/install
View File

@ -22,7 +22,4 @@ done
# Make a place for our horizon config overrides to live
mkdir -p ${SNAP_COMMON}/etc/horizon/local_settings.d
# provide initial openvswitch binaries alternatives
${SNAP}/bin/ovs-alternatives --install
snap-openstack setup # Sets up templates for the first time.

5
snap/hooks/remove
View File

@ -1,5 +0,0 @@
#!/bin/bash
set -ex
# cleanup openvswitch binaries alternatives
${SNAP}/bin/ovs-alternatives --remove

135
snapcraft.yaml
View File

@ -322,50 +322,6 @@ apps:
join:
command: /usr/bin/env python3 ${SNAP}/lib/python3.6/site-packages/cluster/client.py
testpmd:
command: dpdk/bin/testpmd
#plugs:
# - network
# - network-bind
# - network-control
# - network-observe
# - hardware-observe
# #- hugepages-observe
# #- hugepages-control
# - process-control
# - system-observe
# #- dpdk-control
# - kernel-module-control
# - kernel-module-observe
dpdk-devbind:
command: dpdk/sbin/dpdk-devbind
#plugs:
# - network
# - network-bind
# - network-control
# - network-observe
# - hardware-observe
# - system-observe
# #- dpdk-control
# - kernel-module-control
# - kernel-module-observe
dpdk-setup:
command: dpdk/share/dpdk/usertools/dpdk-setup.sh
#plugs:
# - network
# - network-bind
# - network-control
# - network-observe
# - hardware-observe
# #- system-observer
# #- dpdk-control
# - kernel-module-control
# - kernel-module-observe
ovs-alternatives:
command: bin/ovs-alternatives
filebeat:
# this is to avoid conflict with filebeat package
@ -381,23 +337,6 @@ apps:
daemon: simple
parts:
dpdk:
plugin: make
source: http://dpdk.org/git/dpdk-stable
source-type: git
source-tag: v18.11 # LTS Version, comment out or change tag to what version you require.
after:
- patches
- uca-sources
build-packages:
- linux-headers-4.15.0-65-generic
- build-essential
override-build: |
# staging libraries conflict with other parts packages
unset CPPFLAGS
# make V=1 for verbose
make install --directory ${SNAPCRAFT_PART_BUILD} T=x86_64-native-linuxapp-gcc DESTDIR=$SNAPCRAFT_PART_INSTALL/dpdk RTE_KERNELDIR=/usr/src/linux-headers-4.15.0-65-generic # RTE_KERNELDIR is only needed if kernel version is not the same as installed on build host.
# Add Ubuntu Cloud Archive sources.
# Allows us to fetch things such as updated libvirt.
uca-sources:
@ -580,34 +519,7 @@ parts:
prime:
- -*
openvswitch-dpdk:
# TODO: figure out a better way to fetch the version
source: http://openvswitch.org/releases/openvswitch-2.11.1.tar.gz
plugin: autotools
build-packages:
- libssl-dev
- try: [libnuma-dev]
- libcap-ng-dev
- libpcap-dev
- libunbound-dev
- python-all
- python-six
- python-setuptools
stage-packages:
- uuid-runtime
- libunbound2
configflags:
- "--localstatedir=/var/snap/$SNAPCRAFT_PROJECT_NAME/common"
- "--sysconfdir=/var/snap/$SNAPCRAFT_PROJECT_NAME/etc"
- "--with-dpdk=$SNAPCRAFT_STAGE/dpdk"
after:
- patches
- dpdk
organize:
./*: openvswitch-dpdk/
openvswitch:
# TODO: figure out a better way to fetch the version
source: http://openvswitch.org/releases/openvswitch-2.11.1.tar.gz
plugin: autotools
build-packages:
@ -627,53 +539,12 @@ parts:
- "--sysconfdir=/var/snap/$SNAPCRAFT_PROJECT_NAME/etc"
after:
- patches
organize:
./*: openvswitch/
ovs-alternatives:
source: ./snap-wrappers/ovs
plugin: dump
after:
- openvswitch
- openvswitch-dpdk
organize:
ovs-alternatives: bin/ovs-alternatives
ovs-alternatives-init:
plugin: nil
after:
- openvswitch
- openvswitch-dpdk
override-prime: |
# assume we don't have different binaries for the two builds
# Create initial links for openvswitch binaries
for dir in \
bin \
etc/init.d \
sbin \
share/openvswitch/scripts \
usr/bin \
var/snap/${SNAPCRAFT_PROJECT_NAME}/etc/bash_completion.d \
; do
cd ${SNAPCRAFT_PRIME}/openvswitch/
mkdir -p ${SNAPCRAFT_PRIME}/${dir}/real
for file in ${dir}/*; do
ln -fs /snap/${SNAPCRAFT_PROJECT_NAME}/current/bin/ovs-wrapper ${SNAPCRAFT_PRIME}/${dir}/${file##*/}
ln -fs /var/snap/${SNAPCRAFT_PROJECT_NAME}/common/${dir}/${file##*/} ${SNAPCRAFT_PRIME}/${dir}/real/${file##*/}
done
cd -
done
# special file: ovs-lib
# there is no way in POSIX sh to detect if being sourced from another script
ln -fs /snap/${SNAPCRAFT_PROJECT_NAME}/current/share/openvswitch/scripts/real/ovs-lib ${SNAPCRAFT_PRIME}/share/openvswitch/scripts/ovs-lib
snapcraftctl prime
ovs-wrapper:
source: ./snap-wrappers/ovs
plugin: dump
after:
- openvswitch
- openvswitch-dpdk
organize:
ovs-wrapper: bin/ovs-wrapper
@ -682,8 +553,6 @@ parts:
plugin: autotools
after:
- patches
- uca-sources
- dpdk
configflags:
- --prefix=/usr
- "--http-log-path=/var/snap/$SNAPCRAFT_PROJECT_NAME/common/log/nginx-access.log"
@ -819,7 +688,7 @@ parts:
# required to ensure that pathing to files etc works at
# runtime
# * is not used to avoid directory merge conflicts
snap/${SNAPCRAFT_PROJECT_NAME}/current/: ./
snap/microstack/current/: ./
kvm-support:
plugin: nil
@ -906,7 +775,7 @@ parts:
# required to ensure that pathing to files etc works at
# runtime
# * is not used to avoid directory merge conflicts
snap/${SNAPCRAFT_PROJECT_NAME}/current/: ./
snap/microstack/current/: ./
# MySQL
mysql-server:

127
tools/init/init/questions/__init__.py
View File

@ -170,20 +170,12 @@ class NetworkSettings(Question):
"""Write network settings, and """
_type = 'auto'
_question = 'Network settings'
interactive = True
def yes(self, answer):
log.info('Configuring networking ...')
questions = [
network.ExtGateway(),
network.ExtCidr(),
network.OvsDpdk(),
]
for question in questions:
if not self.interactive:
question.interactive = False
question.ask()
network.ExtGateway().ask()
network.ExtCidr().ask()
# Now that we have default or overriden values, setup the
# bridge and write all the proper values into our config
@ -191,13 +183,7 @@ class NetworkSettings(Question):
check('setup-br-ex')
check('snap-openstack', 'setup')
questions = [
network.IpForwarding(),
]
for question in questions:
if not self.interactive:
question.interactive = False
question.ask()
network.IpForwarding().ask()
class OsPassword(ConfigQuestion):
@ -366,9 +352,8 @@ class DatabaseSetup(Question):
# Start keystone-uwsgi. We use snapctl, because systemd
# doesn't yet know about the service.
check('snapctl', 'start', '{SNAP_INSTANCE_NAME}.nginx'.format(**_env))
check('snapctl', 'start',
'{SNAP_INSTANCE_NAME}.keystone-uwsgi'.format(**_env))
check('snapctl', 'start', 'microstack.nginx')
check('snapctl', 'start', 'microstack.keystone-uwsgi')
log.info('Configuring Keystone Fernet Keys ...')
check('snap-openstack', 'launch', 'keystone-manage',
@ -403,12 +388,6 @@ class NovaHypervisor(Question):
_type = 'boolean'
config_key = 'config.services.hypervisor'
@property
def services(self):
return [
'{SNAP_INSTANCE_NAME}.nova-compute'.format(**_env)
]
def yes(self, answer):
log.info('Configuring nova compute hypervisor ...')
@ -422,14 +401,11 @@ class NovaHypervisor(Question):
'microstack', 'compute', endpoint,
'http://{compute_ip}:8774/v2.1'.format(**_env))
for service in self.services:
check('snapctl', 'start', service)
check('snapctl', 'start', 'microstack.nova-compute')
def no(self, answer):
log.info('Disabling nova compute service ...')
for service in self.services:
check('systemctl', 'disable', 'snap.{}'.format(service))
check('systemctl', 'disable', 'snap.microstack.nova-compute')
class NovaControlPlane(Question):
@ -438,17 +414,6 @@ class NovaControlPlane(Question):
_type = 'boolean'
config_key = 'config.services.control-plane'
@property
def services(self):
return [
'{SNAP_INSTANCE_NAME}.nova-api'.format(**_env),
'{SNAP_INSTANCE_NAME}.nova-api-metadata'.format(**_env),
'{SNAP_INSTANCE_NAME}.nova-compute'.format(**_env),
'{SNAP_INSTANCE_NAME}.nova-conductor'.format(**_env),
'{SNAP_INSTANCE_NAME}.nova-scheduler'.format(**_env),
'{SNAP_INSTANCE_NAME}.nova-uwsgi'.format(**_env)
]
def _flavors(self) -> None:
"""Create default flavors."""
@ -498,7 +463,13 @@ class NovaControlPlane(Question):
# out manually, because systemd doesn't know about them yet.
# TODO: parse the output of `snapctl services` to get this
# list automagically.
for service in self.services:
for service in [
'microstack.nova-api',
'microstack.nova-api-metadata',
'microstack.nova-conductor',
'microstack.nova-scheduler',
'microstack.nova-uwsgi',
]:
check('snapctl', 'start', service)
check('snap-openstack', 'launch', 'nova-manage', 'api_db', 'sync')
@ -529,10 +500,14 @@ class NovaControlPlane(Question):
def no(self, answer):
log.info('Disabling nova control plane services ...')
nova_compute = '{SNAP_INSTANCE_NAME}.nova-compute'.format(**_env)
for service in [
'snap.microstack.nova-uwsgi',
'snap.microstack.nova-api',
'snap.microstack.nova-conductor',
'snap.microstack.nova-scheduler',
'snap.microstack.nova-api-metadata']:
for service in self.services.remove(nova_compute) or []:
check('systemctl', 'disable', 'snap.{}'.format(service))
check('systemctl', 'disable', service)
class NeutronControlPlane(Question):
@ -541,16 +516,6 @@ class NeutronControlPlane(Question):
_type = 'boolean'
config_key = 'config.services.control-plane'
@property
def services(self):
return [
'{SNAP_INSTANCE_NAME}.neutron-api'.format(**_env),
'{SNAP_INSTANCE_NAME}.neutron-dhcp-agent'.format(**_env),
'{SNAP_INSTANCE_NAME}.neutron-l3-agent'.format(**_env),
'{SNAP_INSTANCE_NAME}.neutron-metadata-agent'.format(**_env),
'{SNAP_INSTANCE_NAME}.neutron-openvswitch-agent'.format(**_env),
]
def yes(self, answer: str) -> None:
log.info('Configuring Neutron')
@ -568,7 +533,13 @@ class NeutronControlPlane(Question):
'microstack', 'network', endpoint,
'http://{control_ip}:9696'.format(**_env))
for service in self.services:
for service in [
'microstack.neutron-api',
'microstack.neutron-dhcp-agent',
'microstack.neutron-l3-agent',
'microstack.neutron-metadata-agent',
'microstack.neutron-openvswitch-agent',
]:
check('snapctl', 'start', service)
check('snap-openstack', 'launch', 'neutron-db-manage', 'upgrade',
@ -608,16 +579,20 @@ class NeutronControlPlane(Question):
neutron on this machine.
"""
openvswitch_agent = """snap.{SNAP_INSTANCE_NAME}.\
neutron-openvswitch-agent""".format(**_env)
# Make sure that the agent is running.
for service in [openvswitch_agent]:
for service in [
'microstack.neutron-openvswitch-agent',
]:
check('snapctl', 'start', service)
# Disable the other services.
for service in self.services.remove(openvswitch_agent) or []:
check('systemctl', 'disable', 'snap.{}'.format(service))
for service in [
'snap.microstack.neutron-api',
'snap.microstack.neutron-dhcp-agent',
'snap.microstack.neutron-metadata-agent',
'snap.microstack.neutron-l3-agent',
]:
check('systemctl', 'disable', service)
class GlanceSetup(Question):
@ -626,14 +601,6 @@ class GlanceSetup(Question):
_type = 'boolean'
config_key = 'config.services.control-plane'
@property
def services(self):
return [
'{SNAP_INSTANCE_NAME}.glance-api'.format(**_env),
# TODO rename this to glance-registry
'{SNAP_INSTANCE_NAME}.registry'.format(**_env),
]
def _fetch_cirros(self) -> None:
if call('openstack', 'image', 'show', 'cirros'):
@ -677,7 +644,10 @@ class GlanceSetup(Question):
'microstack', 'image', endpoint,
'http://{compute_ip}:9292'.format(**_env))
for service in self.services:
for service in [
'microstack.glance-api',
'microstack.registry', # TODO rename to glance-registery
]:
check('snapctl', 'start', service)
check('snap-openstack', 'launch', 'glance-manage', 'db_sync')
@ -691,8 +661,8 @@ class GlanceSetup(Question):
self._fetch_cirros()
def no(self, answer):
for service in self.services:
check('systemctl', 'disable', 'snap.{}'.format(service))
check('systemctl', 'disable', 'snap.microstack.glance-api')
check('systemctl', 'disable', 'snap.microstack.registry')
class KeyPair(Question):
@ -761,12 +731,6 @@ class PostSetup(Question):
config_key = 'config.post-setup'
@property
def services(self):
return [
'{SNAP_INSTANCE_NAME}.horizon-uwsgi'.format(**_env)
]
def yes(self, answer: str) -> None:
log.info('restarting libvirt and virtlogd ...')
@ -775,8 +739,7 @@ class PostSetup(Question):
restart('*virt*')
# Start horizon
for service in self.services:
check('snapctl', 'start', service)
check('snapctl', 'start', 'microstack.horizon-uwsgi')
check('snapctl', 'set', 'initialized=true')
log.info('Complete. Marked microstack as initialized!')

62
tools/init/init/questions/network.py
View File

@ -1,7 +1,6 @@
from init.config import Env, log
from init.questions.question import Question
from init.shell import check, check_output, restart
from os import path, remove
from init.shell import check, check_output
_env = Env().get_env()
@ -51,62 +50,3 @@ class IpForwarding(Question):
log.info('Setting up ipv4 forwarding...')
check('sysctl', 'net.ipv4.ip_forward=1')
class OvsDpdk(Question):
"""Possibly setup OVS DPDK."""
_type = 'boolean'
_question = 'Do you wish to setup OVS DPDK?'
config_key = 'config.network.ovs-dpdk'
interactive = True
def yes(self, answer: bool):
"""Use ovs-vsctl to setup ovs-dpdk"""
log.info('Setting up OVS DPDK...')
check('snapctl', 'set', 'config.network.ovs-dpdk={}'.format(answer))
check('ovs-wrapper', 'ovs-vsctl', '--no-wait', 'set', 'Open_vSwitch',
'.', 'other_config:dpdk-init=true',
'other_config:dpdk-socket-mem=1024,0',
'other_config:pmd-cpu-mask=0x3')
_path = """{SNAP_COMMON}/etc/neutron/neutron.conf.d/\
neutron-dpdk.conf""".format(**_env)
with open(_path, 'w') as _file:
_file.write("""\
[OVS]
datapath_type = netdev
vhostuser_socket_dir = {SNAP_COMMON}/run/openvswitch
""".format(**_env))
# (re)configure alternatives based on the dpdk answer
check('ovs-alternatives', '--install')
restart('ovs*')
restart('neutron*')
def no(self, answer: bool):
log.info('Setting up OVS...')
check('snapctl', 'set', 'config.network.ovs-dpdk={}'.format(answer))
# only remove config kept on default
check('ovs-wrapper', 'ovs-vsctl', '--no-wait', 'remove',
'Open_vSwitch', '.',
'other_config', 'dpdk-init', 'true',
'other_config', 'pmd-cpu-mask', '0x3',
'dpdk-socket-mem', '1024,0')
_path = """{SNAP_COMMON}/etc/neutron/neutron.conf.d/\
neutron-dpdk.conf""".format(**_env)
if path.exists(_path):
remove(_path)
# (re)configure alternatives based on the dpdk answer
check('ovs-alternatives', '--remove')
check('ovs-alternatives', '--install')
restart('ovs*')
restart('neutron*')

7
tools/init/init/shell.py
View File

@ -35,6 +35,7 @@ import wget
from init.config import Env, log
_env = Env().get_env()
@ -158,11 +159,7 @@ def restart(service: str) -> None:
e.g. *rabbit*
"""
check(
'systemctl',
'restart',
'snap.{SNAP_INSTANCE_NAME}.{SERVICE}'.format(**_env, SERVICE=service)
)
check('systemctl', 'restart', 'snap.microstack.{}'.format(service))
def disable(service: str) -> None: