Michele Baldessari 06b66a8094 Add ERL args parameter for rabbit and set the busy wait threshold to none
Additional argument to be able to customize the ERL command line
arguments. This allows the operator to specify some arguments to the
Erlang VM. By default we are now passing '+sbwt none' to set the
the erlang scheduler busy wait threshold. This threshold determines how
long schedulers are to busy wait when running out of work before going
to sleep. On most of our deployments where rabbitmq shares a node with
a lot of other services it makes little sense to busy wait when out of
work in order to improve latency as other processes are more likely to
fail to do work due to CPUs being overused by rabbit,

We have measured a three-fold reduction of CPU usage with this option
with no other observable impact.

We can observe the change in the beam.smp params:
root         346  0.0  0.0  11680  1468 ?        S    18:29   0:00  \_ /bin/sh /usr/sbin/rabbitmq-server
root         367  0.0  0.0  81940  2060 ?        S    18:29   0:00      \_ su rabbitmq -s /bin/sh -c /usr/lib/rabbitmq/bin/rabbitmq-server
rabbitmq     372  0.0  0.0   9672  1520 ?        Ss   18:29   0:00          \_ /bin/sh /usr/lib/rabbitmq/bin/rabbitmq-server
rabbitmq     561  2.6  0.5 2365936 122036 ?      Sl   18:29   1:04              \_ /usr/lib64/erlang/erts-7.3.1.4/bin/beam.smp -W w -A 64 -K true -P 1048576 -K true -sbwt none -B i -- -root /usr/lib64/erlang -progname erl -- -home /var/lib/rabbitmq -- -pa /usr/lib/rabbitmq/lib/rabbitmq_server-3.6.15/ebin -noshell -noinput -s rabbit boot -sname rabbit@controller-0 -boot start_sasl -config /etc/rabbitmq/rabbitmq -kernel inet_default_connect_options [{nodelay,true}] -kernel inet_default_connect_options [{nodelay,true}] -sasl errlog_type error -sasl sasl_error_logger false -rabbit error_logger {file,"/var/log/rabbitmq/rabbit@controller-0.log"} -rabbit sasl_error_logger {file,"/var/log/rabbitmq/rabbit@controller-0-sasl.log"} -rabbit enabled_plugins_file "/etc/rabbitmq/enabled_plugins" -rabbit plugins_dir "/usr/lib/rabbitmq/plugins:/usr/lib/rabbitmq/lib/rabbitmq_server-3.6.15/plugins" -rabbit plugins_expand_dir "/var/lib/rabbitmq/mnesia/rabbit@controller-0-plugins-expand" -os_mon start_cpu_sup false -os_mon start_disksup false -os_mon start_memsup false -mnesia dir "/var/lib/rabbitmq/mnesia/rabbit@controller-0"
rabbitmq     703  0.0  0.0  11588   448 ?        Ss   18:29   0:00                  \_ inet_gethost 4
rabbitmq     704  0.0  0.0  13712   708 ?        S    18:29   0:00                      \_ inet_gethost 4

Change-Id: I0ce91ed9132afe305c60036837c702c2611fa7c2
2018-09-17 18:02:58 +02:00

194 lines
6.9 KiB
YAML

heat_template_version: rocky
description: >
RabbitMQ service configured with Puppet
parameters:
ServiceData:
default: {}
description: Dictionary packing service data
type: json
ServiceNetMap:
default: {}
description: Mapping of service_name -> network name. Typically set
via parameter_defaults in the resource registry. This
mapping overrides those in ServiceNetMapDefaults.
type: json
DefaultPasswords:
default: {}
type: json
RoleName:
default: ''
description: Role name on which the service is applied
type: string
RoleParameters:
default: {}
description: Parameters specific to the role
type: json
EndpointMap:
default: {}
description: Mapping of service endpoint -> protocol. Typically set
via parameter_defaults in the resource registry.
type: json
RabbitUserName:
default: guest
description: The username for RabbitMQ
type: string
RabbitPassword:
description: The password for RabbitMQ
type: string
hidden: true
RabbitFDLimit:
default: 65536
description: Configures RabbitMQ FD limit
type: number
RabbitIPv6:
default: false
description: Enable IPv6 in RabbitMQ
type: boolean
RabbitCookie:
type: string
default: ''
hidden: true
RabbitHAQueues:
description:
The number of HA queues to be configured in rabbit. The default is -1 which
translates to "ha-mode all". The special value 0 will be automatically
overridden to CEIL(N/2) where N is the number of nodes running rabbitmq.
default: 0
type: number
RabbitNetTickTime:
description:
The number of seconds to configure the value of the erlang
net_ticktime kernel variable.
default: 15
type: number
RabbitAdditionalErlArgs:
description:
Additional parameters passed to the Erlang subsystem. The string
needs to be enclosed in quotes twice. We default to +sbwt none
in order to have the erlang vm be less busy on spinlocks, but
we allow a simple way of overriding it.
default: "'+sbwt none'"
type: string
MonitoringSubscriptionRabbitmq:
default: 'overcloud-rabbitmq'
type: string
EnableInternalTLS:
type: boolean
default: false
conditions:
internal_tls_enabled: {equals: [{get_param: EnableInternalTLS}, true]}
outputs:
role_data:
description: Role data for the RabbitMQ role.
value:
service_name: rabbitmq
monitoring_subscription: {get_param: MonitoringSubscriptionRabbitmq}
config_settings:
map_merge:
-
rabbitmq::file_limit: {get_param: RabbitFDLimit}
rabbitmq::default_user: {get_param: RabbitUserName}
rabbitmq::default_pass: {get_param: RabbitPassword}
rabbit_ipv6: {get_param: RabbitIPv6}
tripleo.rabbitmq.firewall_rules:
'109 rabbitmq':
dport:
- 4369
- 5672
- 25672
rabbitmq::delete_guest_user: false
rabbitmq::wipe_db_on_cookie_change: true
rabbitmq::port: 5672
rabbitmq::package_provider: yum
rabbitmq::package_source: undef
rabbitmq::repos_ensure: false
rabbitmq::tcp_keepalive: true
rabbitmq_environment:
NODE_PORT: ''
NODE_IP_ADDRESS: ''
RABBITMQ_NODENAME: "rabbit@%{::hostname}"
RABBITMQ_SERVER_ERL_ARGS: '"+K true +P 1048576 -kernel inet_default_connect_options [{nodelay,true}]"'
RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: {get_param: RabbitAdditionalErlArgs}
'export ERL_EPMD_ADDRESS': "%{hiera('rabbitmq::interface')}"
rabbitmq_kernel_variables:
inet_dist_listen_min: '25672'
inet_dist_listen_max: '25672'
net_ticktime: {get_param: RabbitNetTickTime}
rabbitmq_config_variables:
cluster_partition_handling: 'ignore'
queue_master_locator: '<<"min-masters">>'
loopback_users: '[]'
rabbitmq::erlang_cookie:
yaql:
expression: $.data.passwords.where($ != '').first()
data:
passwords:
- {get_param: RabbitCookie}
- {get_param: [DefaultPasswords, rabbit_cookie]}
# NOTE: bind IP is found in hiera replacing the network name with the
# local node IP for the given network; replacement examples
# (eg. for internal_api):
# internal_api -> IP
# internal_api_uri -> [IP]
# internal_api_subnet - > IP/CIDR
rabbitmq::interface:
str_replace:
template:
"%{hiera('$NETWORK')}"
params:
$NETWORK: {get_param: [ServiceNetMap, RabbitmqNetwork]}
rabbitmq::nr_ha_queues: {get_param: RabbitHAQueues}
rabbitmq::ssl: {get_param: EnableInternalTLS}
rabbitmq::ssl_erl_dist: {get_param: EnableInternalTLS}
rabbitmq::ssl_port: 5672
rabbitmq::ssl_depth: 1
rabbitmq::ssl_only: {get_param: EnableInternalTLS}
rabbitmq::ssl_interface:
str_replace:
template:
"%{hiera('$NETWORK')}"
params:
$NETWORK: {get_param: [ServiceNetMap, RabbitmqNetwork]}
# TODO(jaosorior): Remove this once we set a proper default in
# puppet-tripleo
tripleo::profile::base::rabbitmq::enable_internal_tls: {get_param: EnableInternalTLS}
-
if:
- internal_tls_enabled
- generate_service_certificates: true
tripleo::profile::base::rabbitmq::certificate_specs:
service_certificate: '/etc/pki/tls/certs/rabbitmq.crt'
service_key: '/etc/pki/tls/private/rabbitmq.key'
hostname:
str_replace:
template: "%{hiera('fqdn_NETWORK')}"
params:
NETWORK: {get_param: [ServiceNetMap, RabbitmqNetwork]}
principal:
str_replace:
template: "rabbitmq/%{hiera('fqdn_NETWORK')}"
params:
NETWORK: {get_param: [ServiceNetMap, RabbitmqNetwork]}
- {}
step_config: |
include ::tripleo::profile::base::rabbitmq
upgrade_tasks:
- name: Stop rabbitmq service
when: step|int == 2
service: name=rabbitmq-server state=stopped
- name: Start rabbitmq service
when: step|int == 4
service: name=rabbitmq-server state=started
metadata_settings:
if:
- internal_tls_enabled
-
- service: rabbitmq
network: {get_param: [ServiceNetMap, RabbitmqNetwork]}
type: node
- null