From 5018fa8215a2b1ddb709c6f8e8f7119d295f7652 Mon Sep 17 00:00:00 2001 From: Michele Baldessari Date: Wed, 12 Sep 2018 18:18:53 +0200 Subject: [PATCH] Add ERL args parameter for rabbit and set the busy wait threshold to none Additional argument to be able to customize the ERL command line arguments. This allows the operator to specify some arguments to the Erlang VM. By default we are now passing '+sbwt none' to set the the erlang scheduler busy wait threshold. This threshold determines how long schedulers are to busy wait when running out of work before going to sleep. On most of our deployments where rabbitmq shares a node with a lot of other services it makes little sense to busy wait when out of work in order to improve latency as other processes are more likely to fail to do work due to CPUs being overused by rabbit, We have measured a three-fold reduction of CPU usage with this option with no other observable impact. We can observe the change in the beam.smp params: root 346 0.0 0.0 11680 1468 ? S 18:29 0:00 \_ /bin/sh /usr/sbin/rabbitmq-server root 367 0.0 0.0 81940 2060 ? S 18:29 0:00 \_ su rabbitmq -s /bin/sh -c /usr/lib/rabbitmq/bin/rabbitmq-server rabbitmq 372 0.0 0.0 9672 1520 ? Ss 18:29 0:00 \_ /bin/sh /usr/lib/rabbitmq/bin/rabbitmq-server rabbitmq 561 2.6 0.5 2365936 122036 ? Sl 18:29 1:04 \_ /usr/lib64/erlang/erts-7.3.1.4/bin/beam.smp -W w -A 64 -K true -P 1048576 -K true -sbwt none -B i -- -root /usr/lib64/erlang -progname erl -- -home /var/lib/rabbitmq -- -pa /usr/lib/rabbitmq/lib/rabbitmq_server-3.6.15/ebin -noshell -noinput -s rabbit boot -sname rabbit@controller-0 -boot start_sasl -config /etc/rabbitmq/rabbitmq -kernel inet_default_connect_options [{nodelay,true}] -kernel inet_default_connect_options [{nodelay,true}] -sasl errlog_type error -sasl sasl_error_logger false -rabbit error_logger {file,"/var/log/rabbitmq/rabbit@controller-0.log"} -rabbit sasl_error_logger {file,"/var/log/rabbitmq/rabbit@controller-0-sasl.log"} -rabbit enabled_plugins_file "/etc/rabbitmq/enabled_plugins" -rabbit plugins_dir "/usr/lib/rabbitmq/plugins:/usr/lib/rabbitmq/lib/rabbitmq_server-3.6.15/plugins" -rabbit plugins_expand_dir "/var/lib/rabbitmq/mnesia/rabbit@controller-0-plugins-expand" -os_mon start_cpu_sup false -os_mon start_disksup false -os_mon start_memsup false -mnesia dir "/var/lib/rabbitmq/mnesia/rabbit@controller-0" rabbitmq 703 0.0 0.0 11588 448 ? Ss 18:29 0:00 \_ inet_gethost 4 rabbitmq 704 0.0 0.0 13712 708 ? S 18:29 0:00 \_ inet_gethost 4 Change-Id: I0ce91ed9132afe305c60036837c702c2611fa7c2 (cherry picked from commit 06b66a8094b2ab0be844d4bd1996d760a27e0b2b) (cherry picked from commit e8563c007d17ebff38f4ae0bbb41c7cb47908633) --- puppet/services/rabbitmq.yaml | 9 +++++++++ .../notes/rabbitmq-erl-args-9029cf4605d63dd9.yaml | 8 ++++++++ 2 files changed, 17 insertions(+) create mode 100644 releasenotes/notes/rabbitmq-erl-args-9029cf4605d63dd9.yaml diff --git a/puppet/services/rabbitmq.yaml b/puppet/services/rabbitmq.yaml index 7e2c2f8e47..717e16152e 100644 --- a/puppet/services/rabbitmq.yaml +++ b/puppet/services/rabbitmq.yaml @@ -63,6 +63,14 @@ parameters: net_ticktime kernel variable. default: 15 type: number + RabbitAdditionalErlArgs: + description: + Additional parameters passed to the Erlang subsystem. The string + needs to be enclosed in quotes twice. We default to +sbwt none + in order to have the erlang vm be less busy on spinlocks, but + we allow a simple way of overriding it. + default: "'+sbwt none'" + type: string MonitoringSubscriptionRabbitmq: default: 'overcloud-rabbitmq' type: string @@ -104,6 +112,7 @@ outputs: NODE_IP_ADDRESS: '' RABBITMQ_NODENAME: "rabbit@%{::hostname}" RABBITMQ_SERVER_ERL_ARGS: '"+K true +P 1048576 -kernel inet_default_connect_options [{nodelay,true}]"' + RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: {get_param: RabbitAdditionalErlArgs} 'export ERL_EPMD_ADDRESS': "%{hiera('rabbitmq::interface')}" rabbitmq_kernel_variables: inet_dist_listen_min: '25672' diff --git a/releasenotes/notes/rabbitmq-erl-args-9029cf4605d63dd9.yaml b/releasenotes/notes/rabbitmq-erl-args-9029cf4605d63dd9.yaml new file mode 100644 index 0000000000..3d00fd2386 --- /dev/null +++ b/releasenotes/notes/rabbitmq-erl-args-9029cf4605d63dd9.yaml @@ -0,0 +1,8 @@ +--- +other: + - | + A new parameter called 'RabbitAdditionalErlArgs' that specifies additional arguments + to the Erlang VM has been added. It now defaults to "'+sbwt none'" (http://erlang.org/doc/man/erl.html#+sbwt) + This threshold determines how long schedulers are to busy wait when running out of work before going to sleep. + By setting it to none we let the erlang threads go to sleep right away when they do not have any work to + do.