From b21034c4b8389805d35a4c8d311938594231b899 Mon Sep 17 00:00:00 2001 From: Duncan Martin Walker Date: Mon, 3 Feb 2020 16:54:56 +0000 Subject: [PATCH] Exposed config for logstash elasticsearch endpoints The Elasticsearch endpoints in use by logstash are currently hardcoded to localhost in both the logstash monitoring config and the default logstash pipeline template. This commit allows this to be configured, such that (multiple) remote endpoints can be specified for the case that logstash is not co-located with elasticsearch. The default behaviour is unchanged. Change-Id: Id871f201168ff882a5b37677747c3484f908298f --- .../roles/elastic_logstash/defaults/main.yml | 2 ++ .../templates/logstash.yml.j2 | 3 +-- .../templates/logstash-pipelines.yml.j2 | 20 +++++++++---------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/elk_metrics_7x/roles/elastic_logstash/defaults/main.yml b/elk_metrics_7x/roles/elastic_logstash/defaults/main.yml index 7da7af75..92a00fbf 100644 --- a/elk_metrics_7x/roles/elastic_logstash/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_logstash/defaults/main.yml @@ -15,6 +15,8 @@ temp_dir: /var/lib/logstash/tmp logstash_pipelines: "{{lookup('template', 'logstash-pipelines.yml.j2') }}" +logstash_elasticsearch_endpoints: + - "127.0.0.1:{{ elastic_port }}" # Set processor cores fact q_storage: 1 diff --git a/elk_metrics_7x/roles/elastic_logstash/templates/logstash.yml.j2 b/elk_metrics_7x/roles/elastic_logstash/templates/logstash.yml.j2 index fd4ebc1c..8642f536 100644 --- a/elk_metrics_7x/roles/elastic_logstash/templates/logstash.yml.j2 +++ b/elk_metrics_7x/roles/elastic_logstash/templates/logstash.yml.j2 @@ -231,7 +231,7 @@ path.logs: /var/log/logstash xpack.monitoring.enabled: true #xpack.monitoring.elasticsearch.username: logstash_system #xpack.monitoring.elasticsearch.password: password -xpack.monitoring.elasticsearch.hosts: ["http://127.0.0.1:9200"] +xpack.monitoring.elasticsearch.hosts: {{ logstash_elasticsearch_endpoints | to_json }} #xpack.monitoring.elasticsearch.ssl.certificate_authority: [ "/path/to/ca.crt" ] #xpack.monitoring.elasticsearch.ssl.truststore.path: path/to/file #xpack.monitoring.elasticsearch.ssl.truststore.password: password @@ -257,4 +257,3 @@ xpack.monitoring.collection.pipeline.details.enabled: true #xpack.management.elasticsearch.ssl.verification_mode: certificate #xpack.management.elasticsearch.sniffing: false #xpack.management.logstash.poll_interval: 5s - diff --git a/elk_metrics_7x/templates/logstash-pipelines.yml.j2 b/elk_metrics_7x/templates/logstash-pipelines.yml.j2 index 6bc6fcc7..6080b50f 100644 --- a/elk_metrics_7x/templates/logstash-pipelines.yml.j2 +++ b/elk_metrics_7x/templates/logstash-pipelines.yml.j2 @@ -495,7 +495,7 @@ elasticsearch { id => "elasticsearchDocIDOutputPipeline" document_id => "%{[@metadata][fingerprint]}" - hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"] + hosts => {{ logstash_elasticsearch_endpoints | to_json }} sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }} manage_template => {{ (data_node | bool) | lower }} index => "%{[@metadata][beat]}-%{[@metadata][version]}" @@ -504,7 +504,7 @@ elasticsearch { id => "elasticsearchLegacyDocIDOutputPipeline" document_id => "%{[@metadata][fingerprint]}" - hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"] + hosts => {{ logstash_elasticsearch_endpoints | to_json }} sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }} manage_template => {{ (data_node | bool) | lower }} index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}" @@ -513,7 +513,7 @@ elasticsearch { id => "elasticsearchSyslogDocIDOutputPipeline" document_id => "%{[@metadata][fingerprint]}" - hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"] + hosts => {{ logstash_elasticsearch_endpoints | to_json }} sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }} manage_template => {{ (data_node | bool) | lower }} index => "syslog-%{+YYYY.MM.dd}" @@ -522,7 +522,7 @@ elasticsearch { id => "elasticsearchCollectdDocIDOutputPipeline" document_id => "%{[@metadata][fingerprint]}" - hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"] + hosts => {{ logstash_elasticsearch_endpoints | to_json }} sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }} manage_template => {{ (data_node | bool) | lower }} index => "collectd-%{+YYYY.MM.dd}" @@ -531,7 +531,7 @@ elasticsearch { id => "elasticsearchUndefinedDocIDOutputPipeline" document_id => "%{[@metadata][fingerprint]}" - hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"] + hosts => {{ logstash_elasticsearch_endpoints | to_json }} sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }} manage_template => {{ (data_node | bool) | lower }} index => "undefined-%{+YYYY.MM.dd}" @@ -541,7 +541,7 @@ if [@metadata][version] { elasticsearch { id => "elasticsearchOutputPipeline" - hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"] + hosts => {{ logstash_elasticsearch_endpoints | to_json }} sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }} manage_template => {{ (data_node | bool) | lower }} index => "%{[@metadata][beat]}-%{[@metadata][version]}" @@ -549,7 +549,7 @@ } else if [@metadata][beat] { elasticsearch { id => "elasticsearchLegacyOutputPipeline" - hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"] + hosts => {{ logstash_elasticsearch_endpoints | to_json }} sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }} manage_template => {{ (data_node | bool) | lower }} index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}" @@ -557,7 +557,7 @@ } else if "syslog" in [tags] { elasticsearch { id => "elasticsearchSyslogOutputPipeline" - hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"] + hosts => {{ logstash_elasticsearch_endpoints | to_json }} sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }} manage_template => {{ (data_node | bool) | lower }} index => "syslog-%{+YYYY.MM.dd}" @@ -565,7 +565,7 @@ } else if "collectd" in [tags] { elasticsearch { id => "elasticsearchCollectdOutputPipeline" - hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"] + hosts => {{ logstash_elasticsearch_endpoints | to_json }} sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }} manage_template => {{ (data_node | bool) | lower }} index => "collectd-%{+YYYY.MM.dd}" @@ -573,7 +573,7 @@ } else { elasticsearch { id => "elasticsearchUndefinedOutputPipeline" - hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"] + hosts => {{ logstash_elasticsearch_endpoints | to_json }} sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }} manage_template => {{ (data_node | bool) | lower }} index => "undefined-%{+YYYY.MM.dd}"