openstack/nova
Code Issues Proposed changes

libvirt: add support for virtio-net rx/tx queue sizes

Browse Source 
Add support for configuring the `rx_queue_size` and `tx_queue_size`
options in the virtio-net driver by way of nova.conf

Currently, valid values for the ring buffer sizes are are 256, 512,
and 1024.

Configuring rx requires libvirt v2.3 QEMU v2.7.
Configuring tx requires libvirt v3.7 QEMU v2.10.

Signed-off-by: Sahid Orentino Ferdjaoui <sahid.ferdjaoui@redhat.com>
Co-Authored-By: Nicolas Simonds <nic@metacloud.com>
DocImpact Implements: bp libvirt-virtio-set-queue-sizes
Change-Id: Ib6d4a2d6b9072db42d11ecdde0950cf7a7781944
This commit is contained in:
Sahid Orentino Ferdjaoui
2018-04-16 05:09:27 -04:00
parent 8ea168d641
commit a1eca36557
8 changed files with 257 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
nova/virt/libvirt/config.py
View File

@@ -1320,6 +1320,8 @@ class LibvirtConfigGuestInterface(LibvirtConfigGuestDevice):
self.vhostuser_path = None
self.vhostuser_type = None
self.vhost_queues = None
self.vhost_rx_queue_size = None
self.vhost_tx_queue_size = None
self.vif_inbound_peak = None
self.vif_inbound_burst = None
self.vif_inbound_average = None
@@ -1349,10 +1351,16 @@ class LibvirtConfigGuestInterface(LibvirtConfigGuestDevice):
if drv_elem is not None:
if self.vhost_queues is not None:
drv_elem.set('queues', str(self.vhost_queues))
if self.vhost_rx_queue_size is not None:
drv_elem.set('rx_queue_size', str(self.vhost_rx_queue_size))
if self.vhost_tx_queue_size is not None:
drv_elem.set('tx_queue_size', str(self.vhost_tx_queue_size))
if drv_elem.get('name') or drv_elem.get('queues'):
if (drv_elem.get('name') or drv_elem.get('queues') or
drv_elem.get('rx_queue_size') or
drv_elem.get('tx_queue_size')):
# Append the driver element into the dom only if name
# or queues attributes are set.
# or queues or tx/rx attributes are set.
dev.append(drv_elem)
if self.net_type == "ethernet":
@@ -1444,6 +1452,8 @@ class LibvirtConfigGuestInterface(LibvirtConfigGuestDevice):
elif c.tag == 'driver':
self.driver_name = c.get('name')
self.vhost_queues = c.get('queues')
self.vhost_rx_queue_size = c.get('rx_queue_size')
self.vhost_tx_queue_size = c.get('tx_queue_size')
elif c.tag == 'source':
if self.net_type == 'direct':
self.source_dev = c.get('dev')

16
nova/virt/libvirt/designer.py
View File

@@ -25,9 +25,12 @@ from nova.pci import utils as pci_utils
MIN_LIBVIRT_ETHERNET_SCRIPT_PATH_NONE = (1, 3, 3)
def set_vif_guest_frontend_config(conf, mac, model, driver, queues=None):
def set_vif_guest_frontend_config(conf, mac, model, driver, queues,
rx_queue_size):
"""Populate a LibvirtConfigGuestInterface instance
with guest frontend details.
NOTE: @model, @driver, @queues and @rx_queue_size can be None.
"""
conf.mac_addr = mac
if model is not None:
@@ -36,6 +39,8 @@ def set_vif_guest_frontend_config(conf, mac, model, driver, queues=None):
conf.driver_name = driver
if queues is not None:
conf.vhost_queues = queues
if rx_queue_size:
conf.vhost_rx_queue_size = rx_queue_size
def set_vif_host_backend_bridge_config(conf, brname, tapname=None):
@@ -148,14 +153,21 @@ def set_vif_host_backend_direct_config(conf, devname, mode="passthrough"):
conf.model = "virtio"
def set_vif_host_backend_vhostuser_config(conf, mode, path):
def set_vif_host_backend_vhostuser_config(conf, mode, path, rx_queue_size,
tx_queue_size):
"""Populate a LibvirtConfigGuestInterface instance
with host backend details for vhostuser socket.
NOTE: @rx_queue_size and @tx_queue_size can be None
"""
conf.net_type = "vhostuser"
conf.vhostuser_type = "unix"
conf.vhostuser_mode = mode
conf.vhostuser_path = path
if rx_queue_size:
conf.vhost_rx_queue_size = rx_queue_size
if rx_queue_size:
conf.vhost_tx_queue_size = tx_queue_size
def set_vif_bandwidth_config(conf, inst_type):

75
nova/virt/libvirt/vif.py
View File

@@ -39,8 +39,10 @@ from nova import profiler
from nova import utils
from nova.virt.libvirt import config as vconfig
from nova.virt.libvirt import designer
from nova.virt.libvirt import utils as libvirt_utils
from nova.virt import osinfo
LOG = logging.getLogger(__name__)
CONF = nova.conf.CONF
@@ -50,6 +52,13 @@ MIN_LIBVIRT_VHOSTUSER_MQ = (1, 2, 17)
# vlan tag for macvtap passthrough mode on SRIOV VFs
MIN_LIBVIRT_MACVTAP_PASSTHROUGH_VLAN = (1, 3, 5)
# virtio-net.rx_queue_size support
MIN_LIBVIRT_RX_QUEUE_SIZE = (2, 3, 0)
MIN_QEMU_RX_QUEUE_SIZE = (2, 7, 0)
# virtio-net.tx_queue_size support
MIN_LIBVIRT_TX_QUEUE_SIZE = (3, 7, 0)
MIN_QEMU_TX_QUEUE_SIZE = (2, 10, 0)
def is_vif_model_valid_for_virt(virt_type, vif_model):
valid_models = {
@@ -105,6 +114,10 @@ class LibvirtGenericVIFDriver(object):
def get_base_config(self, instance, mac, image_meta,
inst_type, virt_type, vnic_type, host):
# TODO(sahid): We should rewrite it. This method handles too
# many unrelated things. We probably need to have a specific
# virtio, vhost, vhostuser functions.
conf = vconfig.LibvirtConfigGuestInterface()
# Default to letting libvirt / the hypervisor choose the model
model = None
@@ -138,10 +151,30 @@ class LibvirtGenericVIFDriver(object):
vnic_type not in network_model.VNIC_TYPES_SRIOV):
vhost_drv, vhost_queues = self._get_virtio_mq_settings(image_meta,
inst_type)
# TODO(sahid): It seems that we return driver 'vhost' even
# for vhostuser interface where for vhostuser interface
# the driver should be 'vhost-user'. That currently does
# not create any issue since QEMU ignores the driver
# argument for vhostuser interface but we should probably
# fix that anyway. Also we should enforce that the driver
# use vhost and not None.
driver = vhost_drv or driver
rx_queue_size = None
if driver == 'vhost' or driver is None:
# vhost backend only supports update of RX queue size
rx_queue_size, _ = self._get_virtio_queue_sizes(host)
if rx_queue_size:
# TODO(sahid): Specifically force driver to be vhost
# that because if None we don't generate the XML
# driver element needed to set the queue size
# attribute. This can be removed when get_base_config
# will be fixed and rewrite to set the correct
# backend.
driver = 'vhost'
designer.set_vif_guest_frontend_config(
conf, mac, model, driver, vhost_queues)
conf, mac, model, driver, vhost_queues, rx_queue_size)
return conf
@@ -438,7 +471,10 @@ class LibvirtGenericVIFDriver(object):
conf.driver_name = None
mode, sock_path = self._get_vhostuser_settings(vif)
designer.set_vif_host_backend_vhostuser_config(conf, mode, sock_path)
rx_queue_size, tx_queue_size = self._get_virtio_queue_sizes(host)
designer.set_vif_host_backend_vhostuser_config(
conf, mode, sock_path, rx_queue_size, tx_queue_size)
# (vladikr) Not setting up driver and queues for vhostuser
# as queues are not supported in Libvirt until version 1.2.17
if not host.has_min_version(MIN_LIBVIRT_VHOSTUSER_MQ):
@@ -447,6 +483,38 @@ class LibvirtGenericVIFDriver(object):
return conf
def _get_virtio_queue_sizes(self, host):
"""Returns rx/tx queue sizes configured or (None, None)
Based on tx/rx queue sizes configured on host (nova.conf). The
methods check whether the versions of libvirt and QEMU are
corrects.
"""
# TODO(sahid): For vhostuser interface this function is called
# from get_base_config and also from the method reponsible to
# configure vhostuser interface meaning that the logs can be
# duplicated. In future we want to rewrite get_base_config.
rx, tx = CONF.libvirt.rx_queue_size, CONF.libvirt.tx_queue_size
if rx and not host.has_min_version(
MIN_LIBVIRT_RX_QUEUE_SIZE, MIN_QEMU_RX_QUEUE_SIZE):
LOG.warning('Setting RX queue size requires libvirt %s and QEMU '
'%s version or greater.',
libvirt_utils.version_to_string(
MIN_LIBVIRT_RX_QUEUE_SIZE),
libvirt_utils.version_to_string(
MIN_QEMU_RX_QUEUE_SIZE))
rx = None
if tx and not host.has_min_version(
MIN_LIBVIRT_TX_QUEUE_SIZE, MIN_QEMU_TX_QUEUE_SIZE):
LOG.warning('Setting TX queue size requires libvirt %s and QEMU '
'%s version or greater.',
libvirt_utils.version_to_string(
MIN_LIBVIRT_TX_QUEUE_SIZE),
libvirt_utils.version_to_string(
MIN_QEMU_TX_QUEUE_SIZE))
tx = None
return rx, tx
def get_config_ib_hostdev(self, instance, vif, image_meta,
inst_type, virt_type, host):
return self.get_base_hostdev_pci_config(vif)
@@ -485,8 +553,9 @@ class LibvirtGenericVIFDriver(object):
# and rewrite to set the correct backend.
conf.driver_name = None
rx_queue_size, tx_queue_size = self._get_virtio_queue_sizes(host)
designer.set_vif_host_backend_vhostuser_config(
conf, vif.mode, vif.path)
conf, vif.mode, vif.path, rx_queue_size, tx_queue_size)
if not host.has_min_version(MIN_LIBVIRT_VHOSTUSER_MQ):
LOG.debug('Queues are not a vhostuser supported feature.')
conf.vhost_queues = None