Update journalbeat config

The journalbeat configuration has been updated to make it
similar to all other beats. This change updates our config
so that it is functional with the latest journalbeat release.

Change-Id: Ic70a031bdeb57f2f5439763a3bf9f6b7001e6a31
Signed-off-by: Kevin Carter <kevin@cloudnull.com>
This commit is contained in:
Kevin Carter 2019-02-26 22:03:12 -06:00
parent a3afb64654
commit 4490ed3dea
1 changed files with 351 additions and 142 deletions

View File

@ -1,78 +1,62 @@
{% import 'templates/_macros.j2' as elk_macros %}
#======================== Journalbeat Configuration ============================
###################### Journalbeat Configuration Example #########################
# This file is an example configuration file highlighting only the most common
# options. The journalbeat.reference.yml file from the same directory contains all the
# supported options with more comments. You can use it as a reference.
#
# You can find the full configuration reference here:
# https://www.elastic.co/guide/en/beats/journalbeat/index.html
# For more available modules and options, please see the journalbeat.reference.yml sample
# configuration file.
#=========================== Journalbeat inputs =============================
journalbeat.inputs:
# Paths that should be crawled and fetched. Possible values files and directories.
# When setting a directory, all journals under it are merged.
# When empty starts to read from local journal.
- paths: ["/var/log/journal"]
# The number of seconds to wait before trying to read again from journals.
#backoff: 1s
# The maximum number of seconds to wait before attempting to read again from journals.
#max_backoff: 60s
# Position to start reading from journal. Valid values: head, tail, cursor
seek: cursor
# Fallback position if no cursor data is available.
#cursor_seek_fallback: head
# Exact matching for field values of events.
# Matching for nginx entries: "systemd.unit=nginx"
#include_matches: []
# Optional fields that you can specify to add additional information to the
# output. Fields can be scalar values, arrays, dictionaries, or any nested
# combination of these.
#fields:
# env: staging
#========================= Journalbeat global options ============================
journalbeat:
# What position in journald to seek to at start up
# options: cursor, tail, head (defaults to tail)
#seek_position: tail
# Name of the registry file. If a relative path is used, it is considered relative to the
# data path.
registry_file: registry
# If seek_position is set to cursor and seeking to cursor fails
# fall back to this method. If set to none will it will exit
# options: tail, head, none (defaults to tail)
#cursor_seek_fallback: tail
# The number of seconds to wait before trying to read again from journals.
backoff: 10s
# The maximum number of seconds to wait before attempting to read again from journals.
max_backoff: 60s
# Store the cursor of the successfully published events
#write_cursor_state: true
# Position to start reading from all journal. Possible values: head, tail, cursor
seek: head
# Path to the file to store the cursor (defaults to ".journalbeat-cursor-state")
#cursor_state_file: .journalbeat-cursor-state
# How frequently should we save the cursor to disk (defaults to 5s)
cursor_flush_period: 10s
# Path to the file to store the queue of events pending (defaults to ".journalbeat-pending-queue")
#pending_queue.file: .journalbeat-pending-queue
# How frequently should we save the queue to disk (defaults to 1s).
# Pending queue represents the WAL of events queued to be published
# or being published and waiting for acknowledgement. In case of a
# regular restart of journalbeat all the events not yet acknowledged
# will be flushed to disk during the shutdown.
# In case of disaster most probably journalbeat won't get a chance to shutdown
# itself gracefully and this flush period option will serve you as a
# backup creation frequency option.
pending_queue.flush_period: 5s
# Size of the buffered queue for the published and acknowledged messages
#pending_queue.completed_queue_size: 8192
# Lowercase and remove leading underscores, e.g. "_MESSAGE" -> "message"
# (defaults to false)
clean_field_names: true
# All journal entries are strings by default. You can try to convert them to numbers.
# (defaults to false)
#convert_to_numbers: false
# Store all the fields of the Systemd Journal entry under this field
# Can be almost any string suitable to be a field name of an ElasticSearch document.
# Dots can be used to create nested fields.
# Two exceptions:
# - no repeated dots;
# - no trailing dots, e.g. "journal..field_name." will fail
# (defaults to "" hence stores on the upper level of the event)
#move_metadata_to_field: ""
# Specific units to monitor.
#units: ["httpd.service"]
# gather kernel logs when units are provided
kernel: true
# Custom Journal patterns to match on other than UNIT
#match_patterns: ["FIELD=value"]
# Specificies syslog identifiers to monitor.
#identifiers: ["docker"]
# Specify Journal paths to open. You can pass an array of paths to Systemd Journal paths.
# If you want to open Journal from directory just pass an array consisting of one element
# representing the path. See: https://www.freedesktop.org/software/systemd/man/sd_journal_open.html
# By default this setting is empty thus journalbeat will attempt to find all journal files automatically
{% if journal_dir.stat.exists and journal_dir.stat.isdir %}
journal_paths: ["/var/log/journal"]
{% endif %}
#default_type: journal
# Exact matching for field values of events.
# Matching for nginx entries: "systemd.unit=nginx"
#matches: []
#================================ General ======================================
@ -98,12 +82,86 @@ tags:
# sub-dictionary. Default is false.
#fields_under_root: false
# Internal queue size for single events in processing pipeline
#queue_size: 1000
# Internal queue configuration for buffering events to be published.
#queue:
# Queue type by name (default 'mem')
# The memory queue will present all available events (up to the outputs
# bulk_max_size) to the output, the moment the output is ready to server
# another batch of events.
#mem:
# Max number of events the queue can buffer.
#events: 4096
# The internal queue size for bulk events in the processing pipeline.
# Do not modify this value.
#bulk_queue_size: 0
# Hints the minimum number of events stored in the queue,
# before providing a batch of events to the outputs.
# The default value is set to 2048.
# A value of 0 ensures events are immediately available
# to be sent to the outputs.
#flush.min_events: 2048
# Maximum duration after which events are available to the outputs,
# if the number of events stored in the queue is < min_flush_events.
#flush.timeout: 1s
# The spool queue will store events in a local spool file, before
# forwarding the events to the outputs.
#
# Beta: spooling to disk is currently a beta feature. Use with care.
#
# The spool file is a circular buffer, which blocks once the file/buffer is full.
# Events are put into a write buffer and flushed once the write buffer
# is full or the flush_timeout is triggered.
# Once ACKed by the output, events are removed immediately from the queue,
# making space for new events to be persisted.
#spool:
# The file namespace configures the file path and the file creation settings.
# Once the file exists, the `size`, `page_size` and `prealloc` settings
# will have no more effect.
#file:
# Location of spool file. The default value is ${path.data}/spool.dat.
#path: "${path.data}/spool.dat"
# Configure file permissions if file is created. The default value is 0600.
#permissions: 0600
# File size hint. The spool blocks, once this limit is reached. The default value is 100 MiB.
#size: 100MiB
# The files page size. A file is split into multiple pages of the same size. The default value is 4KiB.
#page_size: 4KiB
# If prealloc is set, the required space for the file is reserved using
# truncate. The default value is true.
#prealloc: true
# Spool writer settings
# Events are serialized into a write buffer. The write buffer is flushed if:
# - The buffer limit has been reached.
# - The configured limit of buffered events is reached.
# - The flush timeout is triggered.
#write:
# Sets the write buffer size.
#buffer_size: 1MiB
# Maximum duration after which events are flushed, if the write buffer
# is not full yet. The default value is 1s.
#flush.timeout: 1s
# Number of maximum buffered events. The write buffer is flushed once the
# limit is reached.
#flush.events: 16384
# Configure the on-disk event encoding. The encoding can be changed
# between restarts.
# Valid encodings are: json, ubjson, and cbor.
#codec: cbor
#read:
# Reader flush timeout, waiting for more events to become available, so
# to fill a complete batch, as required by the outputs.
# If flush_timeout is 0, all available events are forwarded to the
# outputs immediately.
# The default value is 0s.
#flush.timeout: 0s
# Sets the maximum number of CPUs that can be executing simultaneously. The
# default is the number of logical CPUs available in the system.
@ -118,8 +176,8 @@ tags:
#
# event -> filter1 -> event1 -> filter2 ->event2 ...
#
# The supported processors are drop_fields, drop_event, include_fields, and
# add_cloud_metadata.
# The supported processors are drop_fields, drop_event, include_fields,
# decode_json_fields, and add_cloud_metadata.
#
# For example, you can use the following processors to keep the fields that
# contain CPU load percentages, but remove the fields that contain CPU ticks
@ -139,12 +197,101 @@ tags:
# equals:
# http.code: 200
#
# The following example enriches each event with metadata from the cloud
# provider about the host machine. It works on EC2, GCE, and DigitalOcean.
# The following example renames the field a to b:
#
#processors:
#- add_cloud_metadata:
#- rename:
# fields:
# - from: "a"
# to: "b"
#
# The following example tokenizes the string into fields:
#
#processors:
#- dissect:
# tokenizer: "%{key1} - %{key2}"
# field: "message"
# target_prefix: "dissect"
#
# The following example enriches each event with metadata from the cloud
# provider about the host machine. It works on EC2, GCE, DigitalOcean,
# Tencent Cloud, and Alibaba Cloud.
#
#processors:
#- add_cloud_metadata: ~
#
# The following example enriches each event with the machine's local time zone
# offset from UTC.
#
#processors:
#- add_locale:
# format: offset
#
# The following example enriches each event with docker metadata, it matches
# given fields to an existing container id and adds info from that container:
#
#processors:
#- add_docker_metadata:
# host: "unix:///var/run/docker.sock"
# match_fields: ["system.process.cgroup.id"]
# match_pids: ["process.pid", "process.ppid"]
# match_source: true
# match_source_index: 4
# match_short_id: false
# cleanup_timeout: 60
# labels.dedot: false
# # To connect to Docker over TLS you must specify a client and CA certificate.
# #ssl:
# # certificate_authority: "/etc/pki/root/ca.pem"
# # certificate: "/etc/pki/client/cert.pem"
# # key: "/etc/pki/client/cert.key"
#
# The following example enriches each event with docker metadata, it matches
# container id from log path available in `source` field (by default it expects
# it to be /var/lib/docker/containers/*/*.log).
#
#processors:
#- add_docker_metadata: ~
#
# The following example enriches each event with host metadata.
#
#processors:
#- add_host_metadata:
# netinfo.enabled: false
#
# The following example enriches each event with process metadata using
# process IDs included in the event.
#
#processors:
#- add_process_metadata:
# match_pids: ["system.process.ppid"]
# target: system.process.parent
#
# The following example decodes fields containing JSON strings
# and replaces the strings with valid JSON objects.
#
#processors:
#- decode_json_fields:
# fields: ["field1", "field2", ...]
# process_array: false
# max_depth: 1
# target: ""
# overwrite_keys: false
processors:
- add_host_metadata: ~
#============================= Elastic Cloud ==================================
# These settings simplify using journalbeat with the Elastic Cloud (https://cloud.elastic.co/).
# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
# `setup.kibana.host` options.
# You can find the `cloud.id` in the Elastic Cloud web UI.
#cloud.id:
# The cloud.auth setting overwrites the `output.elasticsearch.username` and
# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
#cloud.auth:
#================================ Outputs ======================================
@ -160,12 +307,19 @@ tags:
# Scheme and port can be left out and will be set to the default (http and 9200)
# In case you specify and additional path, the scheme is required: http://localhost:9200/path
# IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
#hosts: ["localhost:9200"]
#hosts: localhost:9200
# Enabled ilm (beta) to use index lifecycle management instead daily indices.
#ilm.enabled: false
#ilm.rollover_alias: "journalbeat"
#ilm.pattern: "{now/d}-000001"
# Set gzip compression level.
#compression_level: 0
# Configure escaping html symbols in strings.
#escape_html: true
# Optional protocol and basic auth credentials.
#protocol: "https"
#username: "elastic"
@ -179,9 +333,10 @@ tags:
# Number of workers per Elasticsearch host.
#worker: 1
# Optional index name. The default is "beatname" plus date
# and generates [beatname-]YYYY.MM.DD keys.
#index: "beatname-%{+yyyy.MM.dd}"
# Optional index name. The default is "journalbeat" plus date
# and generates [journalbeat-]YYYY.MM.DD keys.
# In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
#index: "journalbeat-%{[beat.version]}-%{+yyyy.MM.dd}"
# Optional ingest node pipeline. By default no pipeline will be used.
#pipeline: ""
@ -205,47 +360,21 @@ tags:
# The default is 50.
#bulk_max_size: 50
# Configure http request timeout before failing an request to Elasticsearch.
# The number of seconds to wait before trying to reconnect to Elasticsearch
# after a network error. After waiting backoff.init seconds, the Beat
# tries to reconnect. If the attempt fails, the backoff timer is increased
# exponentially up to backoff.max. After a successful connection, the backoff
# timer is reset. The default is 1s.
#backoff.init: 1s
# The maximum number of seconds to wait before attempting to connect to
# Elasticsearch after a network error. The default is 60s.
#backoff.max: 60s
# Configure http request timeout before failing a request to Elasticsearch.
#timeout: 90
# The number of seconds to wait for new events between two bulk API index requests.
# If `bulk_max_size` is reached before this interval expires, addition bulk index
# requests are made.
#flush_interval: 1s
# A template is used to set the mapping in Elasticsearch
# By default template loading is enabled and the template is loaded.
# These settings can be adjusted to load your own template or overwrite existing ones.
# Set to false to disable template loading.
#template.enabled: true
# Template name. By default the template name is beatname.
#template.name: "beatname"
# Path to template file
#template.path: "${path.config}/beatname.template.json"
# Overwrite existing template
#template.overwrite: false
# If set to true, beatname checks the Elasticsearch version at connect time, and if it
# is 2.x, it loads the file specified by the template.versions.2x.path setting. The
# default is true.
#template.versions.2x.enabled: true
# Path to the Elasticsearch 2.x version of the template file.
#template.versions.2x.path: "${path.config}/beatname.template-es2x.json"
# If set to true, beatname checks the Elasticsearch version at connect time, and if it
# is 6.x, it loads the file specified by the template.versions.6x.path setting. The
# default is true.
#template.versions.6x.enabled: true
# Path to the Elasticsearch 6.x version of the template file.
#template.versions.6x.path: "${path.config}/beatname.template-es6x.json"
# Use SSL settings for HTTPS. Default is true.
# Use SSL settings for HTTPS.
#ssl.enabled: true
# Configure SSL verification mode. If `none` is configured, all server hosts
@ -277,6 +406,9 @@ tags:
# Configure curve types for ECDHE based cipher suites
#ssl.curve_types: []
# Configure what types of renegotiation are supported. Valid options are
# never, once, and freely. Default is never.
#ssl.renegotiation: never
#----------------------------- Logstash output ---------------------------------
{{ elk_macros.output_logstash(inventory_hostname, logstash_data_hosts, ansible_processor_count, 'journalbeat') }}
@ -316,9 +448,16 @@ tags:
#username: ''
#password: ''
# Kafka version beatname is assumed to run against. Defaults to the oldest
# supported stable version (currently version 0.8.2.0)
#version: 0.8.2
# Kafka version journalbeat is assumed to run against. Defaults to the "1.0.0".
#version: '1.0.0'
# Configure JSON encoding
#codec.json:
# Pretty print json event
#pretty: false
# Configure escaping html symbols in strings.
#escape_html: true
# Metadata update configuration. Metadata do contain leader information
# deciding which broker to use when publishing.
@ -366,6 +505,10 @@ tags:
# default is gzip.
#compression: gzip
# Set the compression level. Currently only gzip provides a compression level
# between 0 and 9. The default value is chosen by the compression algorithm.
#compression_level: 4
# The maximum permitted size of JSON-encoded messages. Bigger messages will be
# dropped. The default value is 1000000 (bytes). This value should be equal to
# or less than the broker's message.max.bytes.
@ -377,9 +520,6 @@ tags:
# on error.
#required_acks: 1
# The number of seconds to wait for new events between two producer API calls.
#flush_interval: 1s
# The configurable ClientID used for logging, debugging, and auditing
# purposes. The default is "beats".
#client_id: beats
@ -416,11 +556,23 @@ tags:
# Configure curve types for ECDHE based cipher suites
#ssl.curve_types: []
# Configure what types of renegotiation are supported. Valid options are
# never, once, and freely. Default is never.
#ssl.renegotiation: never
#------------------------------- Redis output ----------------------------------
#output.redis:
# Boolean flag to enable or disable the output module.
#enabled: true
# Configure JSON encoding
#codec.json:
# Pretty print json event
#pretty: false
# Configure escaping html symbols in strings.
#escape_html: true
# The list of Redis servers to connect to. If load balancing is enabled, the
# events are distributed to the servers in the list. If one server becomes
# unreachable, the events are distributed to the reachable servers only.
@ -431,8 +583,8 @@ tags:
#port: 6379
# The name of the Redis list or channel the events are published to. The
# default is beatname.
#key: beatname
# default is journalbeat.
#key: journalbeat
# The password to authenticate with. The default is no authentication.
#password:
@ -468,6 +620,17 @@ tags:
# until all events are published. The default is 3.
#max_retries: 3
# The number of seconds to wait before trying to reconnect to Redis
# after a network error. After waiting backoff.init seconds, the Beat
# tries to reconnect. If the attempt fails, the backoff timer is increased
# exponentially up to backoff.max. After a successful connection, the backoff
# timer is reset. The default is 1s.
#backoff.init: 1s
# The maximum number of seconds to wait before attempting to connect to
# Redis after a network error. The default is 60s.
#backoff.max: 60s
# The maximum number of events to bulk in a single Redis request or pipeline.
# The default is 2048.
#bulk_max_size: 2048
@ -513,22 +676,33 @@ tags:
# Configure curve types for ECDHE based cipher suites
#ssl.curve_types: []
# Configure what types of renegotiation are supported. Valid options are
# never, once, and freely. Default is never.
#ssl.renegotiation: never
#------------------------------- File output -----------------------------------
#output.file:
# Boolean flag to enable or disable the output module.
#enabled: true
# Configure JSON encoding
#codec.json:
# Pretty print json event
#pretty: false
# Configure escaping html symbols in strings.
#escape_html: true
# Path to the directory where to save the generated files. The option is
# mandatory.
#path: "/tmp/beatname"
#path: "/tmp/journalbeat"
# Name of the generated files. The default is `beatname` and it generates
# files: `beatname`, `beatname.1`, `beatname.2`, etc.
#filename: beatname
# Name of the generated files. The default is `journalbeat` and it generates
# files: `journalbeat`, `journalbeat.1`, `journalbeat.2`, etc.
#filename: journalbeat
# Maximum size in kilobytes of each file. When this size is reached, and on
# every beatname restart, the files are rotated. The default value is 10240
# every journalbeat restart, the files are rotated. The default value is 10240
# kB.
#rotate_every_kb: 10000
@ -537,41 +711,53 @@ tags:
# default is 7 files.
#number_of_files: 7
# Permissions to use for file creation. The default is 0600.
#permissions: 0600
#----------------------------- Console output ---------------------------------
#output.console:
# Boolean flag to enable or disable the output module.
#enabled: true
# Pretty print json event
#pretty: false
# Configure JSON encoding
#codec.json:
# Pretty print json event
#pretty: false
# Configure escaping html symbols in strings.
#escape_html: true
#================================= Paths ======================================
# The home path for the beatname installation. This is the default base path
# The home path for the journalbeat installation. This is the default base path
# for all other path settings and for miscellaneous files that come with the
# distribution (for example, the sample dashboards).
# If not set by a CLI flag or in the configuration file, the default for the
# home path is the location of the binary.
#path.home:
# The configuration path for the beatname installation. This is the default
# The configuration path for the journalbeat installation. This is the default
# base path for configuration files, including the main YAML configuration file
# and the Elasticsearch template file. If not set by a CLI flag or in the
# configuration file, the default for the configuration path is the home path.
#path.config: ${path.home}
# The data path for the beatname installation. This is the default base path
# for all the files in which beatname needs to store its data. If not set by a
# The data path for the journalbeat installation. This is the default base path
# for all the files in which journalbeat needs to store its data. If not set by a
# CLI flag or in the configuration file, the default for the data path is a data
# subdirectory inside the home path.
#path.data: ${path.home}/data
# The logs path for a beatname installation. This is the default location for
# The logs path for a journalbeat installation. This is the default location for
# the Beat's log files. If not set by a CLI flag or in the configuration file,
# the default for the logs path is a logs subdirectory inside the home path.
#path.logs: ${path.home}/logs
#================================ Keystore ==========================================
# Location of the Keystore containing the keys and their sensitive values.
#keystore.path: "${path.config}/beats.keystore"
#============================== Dashboards =====================================
{{ elk_macros.setup_dashboards('journalbeat') }}
@ -585,3 +771,26 @@ tags:
#================================ Logging ======================================
{{ elk_macros.beat_logging('journalbeat') }}
#============================== Xpack Monitoring ===============================
{{ elk_macros.xpack_monitoring_elasticsearch(inventory_hostname, elasticsearch_data_hosts, ansible_processor_count) }}
#================================ HTTP Endpoint ======================================
# Each beat can expose internal metrics through a HTTP endpoint. For security
# reasons the endpoint is disabled by default. This feature is currently experimental.
# Stats can be access through http://localhost:5066/stats . For pretty JSON output
# append ?pretty to the URL.
# Defines if the HTTP endpoint is enabled.
#http.enabled: false
# The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost.
#http.host: localhost
# Port on which the HTTP endpoint will bind. Default is 5066.
#http.port: 5066
#============================= Process Security ================================
# Enable or disable seccomp system call filtering on Linux. Default is enabled.
#seccomp.enabled: true