9c9efd9eb5
This change removes the {h,q}_mem options in favor of a new variable which
clearly states the upper and lower limits for a given deployment. This change
also makes these options a lot more conservative by default which will allow
the deployment to better run on shared infra.
Change-Id: I169f457198c11edc4881a04df65312f6c4f67feb
Signed-off-by: Kevin Carter <kevin@cloudnull.com>
260 lines
8.9 KiB
Django/Jinja
260 lines
8.9 KiB
Django/Jinja
# Settings file in YAML
|
|
#
|
|
# Settings can be specified either in hierarchical form, e.g.:
|
|
#
|
|
# pipeline:
|
|
# batch:
|
|
# size: 125
|
|
# delay: 5
|
|
#
|
|
# Or as flat keys:
|
|
#
|
|
# pipeline.batch.size: 125
|
|
# pipeline.batch.delay: 5
|
|
#
|
|
# ------------ Node identity ------------
|
|
#
|
|
# Use a descriptive name for the node:
|
|
#
|
|
node.name: {{ inventory_hostname }}
|
|
#
|
|
# If omitted the node name will default to the machine's host name
|
|
#
|
|
# ------------ Data path ------------------
|
|
#
|
|
# Which directory should be used by logstash and its plugins
|
|
# for any persistent needs. Defaults to LOGSTASH_HOME/data
|
|
#
|
|
path.data: /var/lib/logstash
|
|
#
|
|
# ------------ Pipeline Settings --------------
|
|
#
|
|
# The ID of the pipeline.
|
|
#
|
|
# pipeline.id: main
|
|
#
|
|
# Set the number of workers that will, in parallel, execute the filters+outputs
|
|
# stage of the pipeline.
|
|
#
|
|
# This defaults to the number of the host's CPU cores.
|
|
#
|
|
|
|
{% set _d_processors = ((ansible_processor_count | int) * 3) %}
|
|
{% set _processors = ((_d_processors | int) > 0) | ternary(_d_processors, 2) %}
|
|
{% set _t_processors = (_processors | int) + (ansible_processor_count | int) %}
|
|
{% set processors = ((_t_processors | int) > 64) | ternary(64, _t_processors) %}
|
|
pipeline.workers: {{ processors | int }}
|
|
#
|
|
# How many events to retrieve from inputs before sending to filters+workers
|
|
#
|
|
pipeline.batch.size: 256
|
|
#
|
|
# How long to wait in milliseconds while polling for the next event
|
|
# before dispatching an undersized batch to filters+outputs
|
|
#
|
|
pipeline.batch.delay: 64
|
|
#
|
|
# Force Logstash to exit during shutdown even if there are still inflight
|
|
# events in memory. By default, logstash will refuse to quit until all
|
|
# received events have been pushed to the outputs.
|
|
#
|
|
# WARNING: enabling this can lead to data loss during shutdown
|
|
#
|
|
# pipeline.unsafe_shutdown: false
|
|
#
|
|
# ------------ Pipeline Configuration Settings --------------
|
|
#
|
|
# Where to fetch the pipeline configuration for the main pipeline
|
|
#
|
|
# path.config:
|
|
#
|
|
# Pipeline configuration string for the main pipeline
|
|
#
|
|
# config.string:
|
|
#
|
|
# At startup, test if the configuration is valid and exit (dry run)
|
|
#
|
|
# config.test_and_exit: false
|
|
#
|
|
# Periodically check if the configuration has changed and reload the pipeline
|
|
# This can also be triggered manually through the SIGHUP signal
|
|
#
|
|
# config.reload.automatic: false
|
|
#
|
|
# How often to check if the pipeline configuration has changed (in seconds)
|
|
#
|
|
# config.reload.interval: 3s
|
|
#
|
|
# Show fully compiled configuration as debug log message
|
|
# NOTE: --log.level must be 'debug'
|
|
#
|
|
# config.debug: false
|
|
#
|
|
# When enabled, process escaped characters such as \n and \" in strings in the
|
|
# pipeline configuration files.
|
|
#
|
|
# config.support_escapes: false
|
|
#
|
|
# ------------ Module Settings ---------------
|
|
# Define modules here. Modules definitions must be defined as an array.
|
|
# The simple way to see this is to prepend each `name` with a `-`, and keep
|
|
# all associated variables under the `name` they are associated with, and
|
|
# above the next, like this:
|
|
#
|
|
# modules:
|
|
# - name: MODULE_NAME
|
|
# var.PLUGINTYPE1.PLUGINNAME1.KEY1: VALUE
|
|
# var.PLUGINTYPE1.PLUGINNAME1.KEY2: VALUE
|
|
# var.PLUGINTYPE2.PLUGINNAME1.KEY1: VALUE
|
|
# var.PLUGINTYPE3.PLUGINNAME3.KEY1: VALUE
|
|
#
|
|
# Module variable names must be in the format of
|
|
#
|
|
# var.PLUGIN_TYPE.PLUGIN_NAME.KEY
|
|
#
|
|
# modules:
|
|
#
|
|
# ------------ Cloud Settings ---------------
|
|
# Define Elastic Cloud settings here.
|
|
# Format of cloud.id is a base64 value e.g. dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy
|
|
# and it may have an label prefix e.g. staging:dXMtZ...
|
|
# This will overwrite 'var.elasticsearch.hosts' and 'var.kibana.host'
|
|
# cloud.id: <identifier>
|
|
#
|
|
# Format of cloud.auth is: <user>:<pass>
|
|
# This is optional
|
|
# If supplied this will overwrite 'var.elasticsearch.username' and 'var.elasticsearch.password'
|
|
# If supplied this will overwrite 'var.kibana.username' and 'var.kibana.password'
|
|
# cloud.auth: elastic:<password>
|
|
#
|
|
# ------------ Queuing Settings --------------
|
|
#
|
|
# Internal queuing model, "memory" for legacy in-memory based queuing and
|
|
# "persisted" for disk-based acked queueing. Defaults is memory
|
|
#
|
|
queue.type: persisted
|
|
#
|
|
# If using queue.type: persisted, the directory path where the data files will be stored.
|
|
# Default is path.data/queue
|
|
#
|
|
# path.queue:
|
|
#
|
|
# If using queue.type: persisted, the page data files size. The queue data consists of
|
|
# append-only data files separated into pages. Default is 64mb
|
|
#
|
|
# queue.page_capacity: 64mb
|
|
#
|
|
# If using queue.type: persisted, the maximum number of unread events in the queue.
|
|
# Default is 0 (unlimited)
|
|
#
|
|
# queue.max_events: 0
|
|
#
|
|
# If using queue.type: persisted, the total capacity of the queue in number of bytes.
|
|
# If you would like more unacked events to be buffered in Logstash, you can increase the
|
|
# capacity using this setting. Please make sure your disk drive has capacity greater than
|
|
# the size specified here. If both max_bytes and max_events are specified, Logstash will pick
|
|
# whichever criteria is reached first
|
|
# Default is 1024mb or 1gb
|
|
#
|
|
{% if logstash_queue_type == 'memory' %}
|
|
# An in memory queue is being used. The actual size of the queue is 90% of the
|
|
# total memory limit, which is set using 50% of the heap size.
|
|
{% set _memory_queue_size = ((elastic_memory_lower_limit | int) // 2) %}
|
|
{% set _memory_queue_size_buffer = (((_memory_queue_size | int) * 0.1) | int) %}
|
|
queue.max_bytes: {{ (_memory_queue_size | int) - (_memory_queue_size_buffer | int) }}mb
|
|
{% else %}
|
|
queue.max_bytes: {{ logstash_queue_size }}mb
|
|
{% endif %}
|
|
#
|
|
# If using queue.type: persisted, the maximum number of acked events before forcing a checkpoint
|
|
# Default is 1024, 0 for unlimited
|
|
#
|
|
# queue.checkpoint.acks: 1024
|
|
#
|
|
# If using queue.type: persisted, the maximum number of written events before forcing a checkpoint
|
|
# Default is 1024, 0 for unlimited
|
|
#
|
|
# queue.checkpoint.writes: 1024
|
|
#
|
|
# If using queue.type: persisted, the interval in milliseconds when a checkpoint is forced on the head page
|
|
# Default is 1000, 0 for no periodic checkpoint.
|
|
#
|
|
# queue.checkpoint.interval: 1000
|
|
#
|
|
# ------------ Dead-Letter Queue Settings --------------
|
|
# Flag to turn on dead-letter queue.
|
|
#
|
|
# dead_letter_queue.enable: false
|
|
|
|
# If using dead_letter_queue.enable: true, the maximum size of each dead letter queue. Entries
|
|
# will be dropped if they would increase the size of the dead letter queue beyond this setting.
|
|
# Default is 1024mb
|
|
# dead_letter_queue.max_bytes: 1024mb
|
|
|
|
# If using dead_letter_queue.enable: true, the directory path where the data files will be stored.
|
|
# Default is path.data/dead_letter_queue
|
|
#
|
|
# path.dead_letter_queue:
|
|
#
|
|
# ------------ Metrics Settings --------------
|
|
#
|
|
# Bind address for the metrics REST endpoint
|
|
#
|
|
# http.host: "127.0.0.1"
|
|
#
|
|
# Bind port for the metrics REST endpoint, this option also accept a range
|
|
# (9600-9700) and logstash will pick up the first available ports.
|
|
#
|
|
# http.port: 9600-9700
|
|
#
|
|
# ------------ Debugging Settings --------------
|
|
#
|
|
# Options for log.level:
|
|
# * fatal
|
|
# * error
|
|
# * warn
|
|
# * info (default)
|
|
# * debug
|
|
# * trace
|
|
#
|
|
# log.level: info
|
|
path.logs: /var/log/logstash
|
|
#
|
|
# ------------ Other Settings --------------
|
|
#
|
|
# Where to find custom plugins
|
|
# path.plugins: []
|
|
#
|
|
# ---------------------------------- X-Pack ------------------------------------
|
|
# X-Pack Monitoring
|
|
# https://www.elastic.co/guide/en/logstash/current/monitoring-logstash.html
|
|
xpack.monitoring.enabled: true
|
|
#xpack.monitoring.elasticsearch.username: logstash_system
|
|
#xpack.monitoring.elasticsearch.password: password
|
|
xpack.monitoring.elasticsearch.url: ["127.0.0.1:9200"]
|
|
#xpack.monitoring.elasticsearch.ssl.ca: [ "/path/to/ca.crt" ]
|
|
#xpack.monitoring.elasticsearch.ssl.truststore.path: path/to/file
|
|
#xpack.monitoring.elasticsearch.ssl.truststore.password: password
|
|
#xpack.monitoring.elasticsearch.ssl.keystore.path: /path/to/file
|
|
#xpack.monitoring.elasticsearch.ssl.keystore.password: password
|
|
#xpack.monitoring.elasticsearch.ssl.verification_mode: certificate
|
|
xpack.monitoring.elasticsearch.sniffing: {{ elastic_sniffing_enabled | default(false) }}
|
|
xpack.monitoring.collection.interval: 30s
|
|
xpack.monitoring.collection.pipeline.details.enabled: true
|
|
#
|
|
# ------------ X-Pack Settings (not applicable for OSS build)--------------
|
|
# X-Pack Management
|
|
# https://www.elastic.co/guide/en/logstash/current/logstash-centralized-pipeline-management.html
|
|
#xpack.management.enabled: false
|
|
#xpack.management.pipeline.id: ["main", "apache_logs"]
|
|
#xpack.management.elasticsearch.username: logstash_admin_user
|
|
#xpack.management.elasticsearch.password: password
|
|
#xpack.management.elasticsearch.url: ["https://es1:9200", "https://es2:9200"]
|
|
#xpack.management.elasticsearch.ssl.ca: [ "/path/to/ca.crt" ]
|
|
#xpack.management.elasticsearch.ssl.truststore.path: /path/to/file
|
|
#xpack.management.elasticsearch.ssl.truststore.password: password
|
|
#xpack.management.elasticsearch.ssl.keystore.path: /path/to/file
|
|
#xpack.management.elasticsearch.ssl.keystore.password: password
|
|
#xpack.management.elasticsearch.sniffing: {{ elastic_sniffing_enabled | default(false) }}
|
|
#xpack.management.logstash.poll_interval: 5s
|