From f2605e928136f88032ef9b4977a076d5c535987b Mon Sep 17 00:00:00 2001 From: Julia Kreger Date: Wed, 26 Apr 2023 07:34:29 -0700 Subject: [PATCH] Remove use of nomodeset by default The troubleshooting kernel command line option nomodeset unfortunately changes the way framebuffer interactions work with graphics devices which in some cases can result in kernel memory to be used for graphics updates. When this happens on some specific hardware common in rack mount servers with baseboard management controllers, this can cause the memory bus to become locked for a brief time while the graphics update is occuring. This locked memory bus means disk IO can become blocked, and network cards can overflow their buffers resulting in packet loss on top of the latency incurred by the graphics update executing. As such, we've removed the nomodeset option from default usage and added a note describing its removal to the documentation along with a release note. Change-Id: I9084d88c3ec6f13bd64b8707892758fa87dd7f86 --- devstack/lib/ironic | 4 ++-- .../tools/ironic/scripts/cirros-partition.sh | 2 +- doc/source/admin/drivers/redfish.rst | 2 +- doc/source/admin/interfaces/boot.rst | 16 ++++++++++---- .../include/kernel-boot-parameters.inc | 4 ++-- ironic/conf/ilo.py | 2 +- ironic/conf/pxe.py | 2 +- ironic/conf/redfish.py | 2 +- .../unit/drivers/modules/test_image_utils.py | 8 +++---- .../tests/unit/drivers/modules/test_ipxe.py | 2 +- .../remove-nomodset-7a352a9519c1045b.yaml | 22 +++++++++++++++++++ 11 files changed, 48 insertions(+), 18 deletions(-) create mode 100644 releasenotes/notes/remove-nomodset-7a352a9519c1045b.yaml diff --git a/devstack/lib/ironic b/devstack/lib/ironic index 17ba547f11..b81b9f38ba 100644 --- a/devstack/lib/ironic +++ b/devstack/lib/ironic @@ -1793,7 +1793,7 @@ function configure_ironic_conductor { -subj '/O=OpenStack/OU=DevStack Servers' fi - local pxe_params="nofb nomodeset console=${IRONIC_TTY_DEV}" + local pxe_params="nofb console=${IRONIC_TTY_DEV}" pxe_params+=" systemd.journald.forward_to_console=yes" if is_service_enabled tls-proxy; then pxe_params+=" ipa-insecure=1" @@ -1805,7 +1805,7 @@ function configure_ironic_conductor { iniset $IRONIC_CONF_FILE pxe kernel_append_params "$pxe_params" fi - local kernel_append_params="nofb nomodeset console=${IRONIC_TTY_DEV}" + local kernel_append_params="nofb console=${IRONIC_TTY_DEV}" kernel_append_params+=" systemd.journald.forward_to_console=yes" # NOTE(dtantsur): avoid setting ipa-insecure for redfish, we have a way to diff --git a/devstack/tools/ironic/scripts/cirros-partition.sh b/devstack/tools/ironic/scripts/cirros-partition.sh index facf9b0304..38e1764c4b 100755 --- a/devstack/tools/ironic/scripts/cirros-partition.sh +++ b/devstack/tools/ironic/scripts/cirros-partition.sh @@ -18,7 +18,7 @@ IRONIC_TTY_DEV=${IRONIC_TTY_DEV:-ttyS0,115200} # rdroot : boot from the ramdisk present on the root partition instead of # mounting the root partition. # dslist : disable Nova metadata support, it takes a long time on boot. -KARGS=${KARGS:-nofb nomodeset vga=normal console=${IRONIC_TTY_DEV} rdroot dslist=configdrive} +KARGS=${KARGS:-nofb vga=normal console=${IRONIC_TTY_DEV} rdroot dslist=configdrive} workdir=$(mktemp -d) root_mp=$workdir/root diff --git a/doc/source/admin/drivers/redfish.rst b/doc/source/admin/drivers/redfish.rst index 063dd1fe5c..7719088387 100644 --- a/doc/source/admin/drivers/redfish.rst +++ b/doc/source/admin/drivers/redfish.rst @@ -214,7 +214,7 @@ user-specified kernel command line parameters. .. code-block:: bash baremetal node set node-0 \ - --driver-info kernel_append_params="nofb nomodeset vga=normal" + --driver-info kernel_append_params="nofb vga=normal" .. note:: The ``driver_info`` field is supported starting with the Xena release. diff --git a/doc/source/admin/interfaces/boot.rst b/doc/source/admin/interfaces/boot.rst index b9f0bda3e2..a4b4cf2a35 100644 --- a/doc/source/admin/interfaces/boot.rst +++ b/doc/source/admin/interfaces/boot.rst @@ -49,7 +49,7 @@ configuration option: .. code-block:: ini [pxe] - kernel_append_params = nofb nomodeset vga=normal + kernel_append_params = nofb vga=normal .. note:: The option was called ``pxe_append_params`` before the Xena cycle. @@ -59,9 +59,9 @@ Per-node and per-instance overrides are also possible, for example: .. code-block:: bash baremetal node set node-0 \ - --driver-info kernel_append_params="nofb nomodeset vga=normal" + --driver-info kernel_append_params="nofb vga=normal" baremetal node set node-0 \ - --instance-info kernel_append_params="nofb nomodeset vga=normal" + --instance-info kernel_append_params="nofb vga=normal" Starting with the Zed cycle, you can combine the parameters from the configuration and from the node using the special ``%default%`` syntax: @@ -74,13 +74,21 @@ configuration and from the node using the special ``%default%`` syntax: Together with the configuration above, the following parameters will be appended to the kernel command line:: - nofb nomodeset vga=normal console=ttyS0,115200n8 + nofb vga=normal console=ttyS0,115200n8 .. note:: Ironic does not do any de-duplication of the resulting kernel parameters. Both kernel itself and dracut seem to give priority to the last instance of the same parameter. +.. warning:: + Previously our documentation listed the Linux kernel parameter + ``nomodeset`` as an option. This option is intended for troubleshooting, + and can greatly degrade performance with Matrox/Aspeed BMC Graphics + controllers which is very commonly used on physical servers. The + performance degredation can greatly reduce IO capacity upon every + console graphics update being written to the screen. + Common options -------------- diff --git a/doc/source/install/include/kernel-boot-parameters.inc b/doc/source/install/include/kernel-boot-parameters.inc index f0da59d601..82b4ecc910 100644 --- a/doc/source/install/include/kernel-boot-parameters.inc +++ b/doc/source/install/include/kernel-boot-parameters.inc @@ -91,7 +91,7 @@ respectively. [pxe] # Additional append parameters for baremetal PXE boot. - kernel_append_params = nofb nomodeset vga=normal console=ttyS0,115200n8 + kernel_append_params = nofb vga=normal console=ttyS0,115200n8 * For node web console configuration is similar with the addition of ``ttyX`` @@ -100,7 +100,7 @@ respectively. [pxe] # Additional append parameters for baremetal PXE boot. - kernel_append_params = nofb nomodeset vga=normal console=tty0 console=ttyS0,115200n8 + kernel_append_params = nofb vga=normal console=tty0 console=ttyS0,115200n8 For detailed information on how to add consoles see the reference documents `kernel params`_ and `serial console`_. diff --git a/ironic/conf/ilo.py b/ironic/conf/ilo.py index 197378ce75..6d1c52dd20 100644 --- a/ironic/conf/ilo.py +++ b/ironic/conf/ilo.py @@ -112,7 +112,7 @@ opts = [ 'This setting must be set to the octal number ' 'representation, meaning starting with ``0o``.')), cfg.StrOpt('kernel_append_params', - default='nofb nomodeset vga=normal', + default='nofb vga=normal', mutable=True, help=_('Additional kernel parameters to pass down to the ' 'instance kernel. These parameters can be consumed by ' diff --git a/ironic/conf/pxe.py b/ironic/conf/pxe.py index d967128248..e4c912192d 100644 --- a/ironic/conf/pxe.py +++ b/ironic/conf/pxe.py @@ -24,7 +24,7 @@ from ironic.common.i18n import _ opts = [ cfg.StrOpt('kernel_append_params', deprecated_name='pxe_append_params', - default='nofb nomodeset vga=normal', + default='nofb vga=normal', mutable=True, help=_('Additional append parameters for baremetal PXE boot.')), cfg.StrOpt('default_ephemeral_format', diff --git a/ironic/conf/redfish.py b/ironic/conf/redfish.py index 3cc9fe0159..68aa961414 100644 --- a/ironic/conf/redfish.py +++ b/ironic/conf/redfish.py @@ -64,7 +64,7 @@ opts = [ 'auto-expire. Applies only when `use_swift` is ' 'enabled.')), cfg.StrOpt('kernel_append_params', - default='nofb nomodeset vga=normal', + default='nofb vga=normal', mutable=True, help=_('Additional kernel parameters to pass down to the ' 'instance kernel. These parameters can be consumed by ' diff --git a/ironic/tests/unit/drivers/modules/test_image_utils.py b/ironic/tests/unit/drivers/modules/test_image_utils.py index b6c572125d..fbb6318fe5 100644 --- a/ironic/tests/unit/drivers/modules/test_image_utils.py +++ b/ironic/tests/unit/drivers/modules/test_image_utils.py @@ -583,7 +583,7 @@ class RedfishImageUtilsTestCase(db_base.DbTestCase): mock_create_boot_iso.assert_called_once_with( mock.ANY, mock.ANY, 'http://kernel/img', 'http://ramdisk/img', boot_mode='uefi', esp_image_href='http://bootloader/img', - kernel_params='nofb nomodeset vga=normal', + kernel_params='nofb vga=normal', root_uuid='1be26c0b-03f2-4d2e-ae87-c02d7f33c123', inject_files=None) @@ -604,7 +604,7 @@ class RedfishImageUtilsTestCase(db_base.DbTestCase): mock_create_boot_iso.assert_called_once_with( mock.ANY, mock.ANY, 'http://kernel/img', 'http://ramdisk/img', boot_mode='uefi', esp_image_href=None, - kernel_params='nofb nomodeset vga=normal', + kernel_params='nofb vga=normal', root_uuid='1be26c0b-03f2-4d2e-ae87-c02d7f33c123', inject_files=None) @@ -633,7 +633,7 @@ class RedfishImageUtilsTestCase(db_base.DbTestCase): mock_create_boot_iso.assert_called_once_with( mock.ANY, mock.ANY, 'http://kernel/img', 'http://ramdisk/img', boot_mode='bios', esp_image_href=None, - kernel_params='nofb nomodeset vga=normal', + kernel_params='nofb vga=normal', root_uuid='1be26c0b-03f2-4d2e-ae87-c02d7f33c123', inject_files=None) @@ -702,7 +702,7 @@ class RedfishImageUtilsTestCase(db_base.DbTestCase): mock_create_boot_iso.assert_called_once_with( mock.ANY, mock.ANY, 'http://kernel/img', 'http://ramdisk/img', boot_mode='uefi', esp_image_href=None, - kernel_params=f'nofb nomodeset vga=normal {kernel_params}', + kernel_params=f'nofb vga=normal {kernel_params}', root_uuid='1be26c0b-03f2-4d2e-ae87-c02d7f33c123', inject_files=None) diff --git a/ironic/tests/unit/drivers/modules/test_ipxe.py b/ironic/tests/unit/drivers/modules/test_ipxe.py index ef37e3fc40..02d254b5a3 100644 --- a/ironic/tests/unit/drivers/modules/test_ipxe.py +++ b/ironic/tests/unit/drivers/modules/test_ipxe.py @@ -813,7 +813,7 @@ class iPXEBootTestCase(db_base.DbTestCase): 'aki_path': 'http://192.1.2.3:1234/' + uuid + '/kernel', 'ari_path': 'http://192.1.2.3:1234/' + uuid + '/ramdisk', 'ramdisk_opts': 'cat meow', - 'pxe_append_params': 'nofb nomodeset vga=normal ipa-debug=1 ' + 'pxe_append_params': 'nofb vga=normal ipa-debug=1 ' 'ipa-global-request-' 'id=' + task.context.request_id, 'tftp_server': mock.ANY, diff --git a/releasenotes/notes/remove-nomodset-7a352a9519c1045b.yaml b/releasenotes/notes/remove-nomodset-7a352a9519c1045b.yaml new file mode 100644 index 0000000000..26d10c2623 --- /dev/null +++ b/releasenotes/notes/remove-nomodset-7a352a9519c1045b.yaml @@ -0,0 +1,22 @@ +--- +upgrade: + - | + The defaults for ``kernel_append_params`` have had the Linux kernel + command line parameter ``nomodeset`` removed from the defaults for the + ``kernel_append_params`` settings. The ``nomodeset`` option is for + troubleshooting and changes the behavior of the graphics interface + such that memory can be locked upon graphical updates on physical + servers with BMC graphical interfaces, which results in spikes in + latency and packet loss whenever graphics updates occur. Operators + may add the option to their local configuration, but should be aware + that large image transfers or other high IO operations can be impacted. +fixes: + - | + Agents deploying on physical servers with default kernel arguments were + suspetible to packet loss if a Matrox VGA/Aspeed BMC Graphics interface + is present on the machine. The defaults have been changed to remove + the use of the ``nomodeset`` kernel command line parameter which + should only be used for troubleshooting as it has been determined + that the memory updates can lock all of the kernel memory upon any + console graphics update which can negatively impact IO for Networking + or Disk interactions.