From 17d8332604e2337b3f2ab8ec8229097ec4174f73 Mon Sep 17 00:00:00 2001
From: Bartosz Bezak <bartosz@stackhpc.com>
Date: Tue, 7 Jul 2020 13:43:18 +0000
Subject: [PATCH] Logstash 6 support
Co-Authored-By: Doug Szumski <doug@stackhpc.com>
Closes-Bug: #1884090
Depends-On: https://review.opendev.org/#/c/736768
Change-Id: If2d0dd1739e484b14e3c15a185a236918737b0ab
---
ansible/roles/monasca/defaults/main.yml | 3 +++
.../monasca-log-metrics/log-metrics.conf.j2 | 11 +++++------
.../monasca-log-metrics/monasca-log-metrics.json.j2 | 2 +-
.../monasca-log-persister/log-persister.conf.j2 | 6 +++---
.../monasca-log-persister.json.j2 | 2 +-
.../monasca-log-transformer/log-transformer.conf.j2 | 9 ++++-----
.../monasca-log-transformer.json.j2 | 2 +-
releasenotes/notes/bug-1884090-c40ff140d5514108.yaml | 12 ++++++++++++
8 files changed, 30 insertions(+), 17 deletions(-)
create mode 100644 releasenotes/notes/bug-1884090-c40ff140d5514108.yaml
diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml
index 18c5624a28..2b0f5f45a2 100644
--- a/ansible/roles/monasca/defaults/main.yml
+++ b/ansible/roles/monasca/defaults/main.yml
@@ -251,16 +251,19 @@ monasca_log_transformer_default_volumes:
- "/etc/localtime:/etc/localtime:ro"
- "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
- "kolla_logs:/var/log/kolla"
+ - "monasca_log_transformer_data:/var/lib/logstash"
monasca_log_persister_default_volumes:
- "{{ node_config_directory }}/monasca-log-persister/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"
- "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
- "kolla_logs:/var/log/kolla"
+ - "monasca_log_persister_data:/var/lib/logstash"
monasca_log_metrics_default_volumes:
- "{{ node_config_directory }}/monasca-log-metrics/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"
- "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
- "kolla_logs:/var/log/kolla"
+ - "monasca_log_metrics_data:/var/lib/logstash"
monasca_thresh_default_volumes:
- "{{ node_config_directory }}/monasca-thresh/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"
diff --git a/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 b/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2
index a237d61a7b..bb05e9d488 100644
--- a/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2
+++ b/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2
@@ -9,11 +9,11 @@
input {
kafka {
- zk_connect => "{{ monasca_zookeeper_servers }}"
- topic_id => "{{ monasca_transformed_logs_topic }}"
+ bootstrap_servers => "{{ monasca_kafka_servers }}"
+ topics => ["{{ monasca_transformed_logs_topic }}"]
group_id => "log_metrics"
- consumer_id => "log_metrics_{{ ansible_hostname }}"
consumer_threads => "{{ monasca_log_pipeline_threads }}"
+ codec => json
}
}
@@ -42,7 +42,7 @@ filter {
# Convert the timestamp of the event to milliseconds since epoch.
ruby {
- code => "event['metric']['timestamp'] = event['@timestamp'].to_i * 1000"
+ code => "event.set('[metric][timestamp]', event.get('[@timestamp]').to_i*1000)"
}
# Clean up any fields which aren't required from the new metric to save space
@@ -68,9 +68,8 @@ filter {
output {
kafka {
+ codec => json
bootstrap_servers => "{{ monasca_kafka_servers }}"
topic_id => "{{ monasca_metrics_topic }}"
- client_id => "log_metrics_{{ ansible_hostname }}"
- workers => {{ monasca_log_pipeline_threads|int }}
}
}
diff --git a/ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2 b/ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2
index 287ffcd975..7c605aeab7 100644
--- a/ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2
+++ b/ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2
@@ -1,5 +1,5 @@
{
- "command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-metrics.log -f /etc/logstash/conf.d/log-metrics.conf",
+ "command": "/usr/share/logstash/bin/logstash --path.settings /etc/logstash/ --log.format json --path.logs /var/log/kolla/logstash/monasca-log-metrics -f /etc/logstash/conf.d/log-metrics.conf",
"config_files": [
{
"source": "{{ container_config_directory }}/log-metrics.conf",
diff --git a/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 b/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2
index 6eb9219b4c..732c07e5a0 100644
--- a/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2
+++ b/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2
@@ -2,11 +2,11 @@
input {
kafka {
- zk_connect => "{{ monasca_zookeeper_servers }}"
- topic_id => "{{ monasca_transformed_logs_topic }}"
+ bootstrap_servers => "{{ monasca_kafka_servers }}"
+ topics => ["{{ monasca_transformed_logs_topic }}"]
group_id => "log_persister"
- consumer_id => "log_persister_{{ ansible_hostname }}"
consumer_threads => "{{ monasca_log_pipeline_threads }}"
+ codec => json
}
}
diff --git a/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2 b/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2
index fa99a35d20..365d293dde 100644
--- a/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2
+++ b/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2
@@ -1,5 +1,5 @@
{
- "command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-persister.log -f /etc/logstash/conf.d/log-persister.conf",
+ "command": "/usr/share/logstash/bin/logstash --path.settings /etc/logstash/ --log.format json --path.logs /var/log/kolla/logstash/monasca-log-persister -f /etc/logstash/conf.d/log-persister.conf",
"config_files": [
{
"source": "{{ container_config_directory }}/log-persister.conf",
diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2
index 494bf5257f..b637925375 100644
--- a/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2
+++ b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2
@@ -3,11 +3,11 @@
input {
kafka {
- zk_connect => "{{ monasca_zookeeper_servers }}"
- topic_id => "{{ monasca_raw_logs_topic }}"
+ bootstrap_servers => "{{ monasca_kafka_servers }}"
+ topics => ["{{ monasca_raw_logs_topic }}"]
group_id => "log_transformer"
- consumer_id => "log_transformer_{{ ansible_hostname }}"
consumer_threads => "{{ monasca_log_pipeline_threads }}"
+ codec => json
}
}
@@ -39,9 +39,8 @@ filter {
output {
kafka {
+ codec => json
bootstrap_servers => "{{ monasca_kafka_servers }}"
topic_id => "{{ monasca_transformed_logs_topic }}"
- client_id => "log_transformer_{{ ansible_hostname }}"
- workers => {{ monasca_log_pipeline_threads|int }}
}
}
diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2
index 733eeedad9..6d5cee36ee 100644
--- a/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2
+++ b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2
@@ -1,5 +1,5 @@
{
- "command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf",
+ "command": "/usr/share/logstash/bin/logstash --path.settings /etc/logstash/ --log.format json --path.logs /var/log/kolla/logstash/monasca-log-transformer -f /etc/logstash/conf.d/log-transformer.conf",
"config_files": [
{
"source": "{{ container_config_directory }}/log-transformer.conf",
diff --git a/releasenotes/notes/bug-1884090-c40ff140d5514108.yaml b/releasenotes/notes/bug-1884090-c40ff140d5514108.yaml
new file mode 100644
index 0000000000..8d9d6128d1
--- /dev/null
+++ b/releasenotes/notes/bug-1884090-c40ff140d5514108.yaml
@@ -0,0 +1,12 @@
+---
+upgrade:
+ - |
+ When deploying Monasca with Logstash 6, any
+ custom Logstash 2 configuration for Monasca will need to be updated to
+ work with Logstash 6. Please consult the `documentation
+ <https://www.elastic.co/guide/en/logstash/master/upgrading-logstash.html>`__.
+fixes:
+ - |
+ When deploying Elasticsearch 6, Logstash 2 was
+ deployed by default which is not compatible with Elasticsearch 6. Logstash
+ 6 is now deployed by default.