From c2e08be414b45156b189c4923bb587596d07ef6b Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Tue, 14 Jan 2020 14:15:40 +0000 Subject: [PATCH] Upgrade service configuration for ELK 7 ELK 7 requires some minor changes from the existing ELK 6 config. Depends-On: Icfa3db5788b25f70ee75411dbaf20d8d4a6a734b Change-Id: I9815d202a77da0477aea43d714a5def8a24724fa --- .../templates/elasticsearch.yml.j2 | 3 +- ansible/roles/kibana/templates/kibana.json.j2 | 2 +- ansible/roles/kibana/templates/kibana.yml.j2 | 2 +- ...ade-elk-to-7-release-c559d4c5dff05d2f.yaml | 7 +++ tests/test-prometheus-efk.sh | 43 +++++++++++++++++-- 5 files changed, 50 insertions(+), 7 deletions(-) create mode 100644 releasenotes/notes/upgrade-elk-to-7-release-c559d4c5dff05d2f.yaml diff --git a/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2 b/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2 index ec4ca41bd8..1f6f944218 100644 --- a/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2 +++ b/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2 @@ -5,9 +5,10 @@ node.name: "{{ 'api' | kolla_address | put_address_in_context('url') }}" network.host: "{{ 'api' | kolla_address | put_address_in_context('url') }}" cluster.name: "{{ elasticsearch_cluster_name }}" +cluster.initial_master_nodes: [{% for host in groups['elasticsearch'] %}"{{ 'api' | kolla_address(host) }}"{% if not loop.last %},{% endif %}{% endfor %}] node.master: true node.data: true -discovery.zen.ping.unicast.hosts: [{% for host in groups['elasticsearch'] %}"{{ 'api' | kolla_address(host) | put_address_in_context('url') }}"{% if not loop.last %},{% endif %}{% endfor %}] +discovery.seed_hosts: [{% for host in groups['elasticsearch'] %}"{{ 'api' | kolla_address(host) | put_address_in_context('url') }}"{% if not loop.last %},{% endif %}{% endfor %}] discovery.zen.minimum_master_nodes: {{ minimum_master_nodes }} http.port: {{ elasticsearch_port }} diff --git a/ansible/roles/kibana/templates/kibana.json.j2 b/ansible/roles/kibana/templates/kibana.json.j2 index f2dfaa9573..2ceb5493cc 100644 --- a/ansible/roles/kibana/templates/kibana.json.j2 +++ b/ansible/roles/kibana/templates/kibana.json.j2 @@ -1,5 +1,5 @@ { - "command": "/usr/share/kibana/bin/kibana", + "command": "/usr/share/kibana/bin/kibana --config /etc/kibana/kibana.yml", "config_files": [ { "source": "{{ container_config_directory }}/kibana.yml", diff --git a/ansible/roles/kibana/templates/kibana.yml.j2 b/ansible/roles/kibana/templates/kibana.yml.j2 index c5eb81cbe2..3cc3ecac53 100644 --- a/ansible/roles/kibana/templates/kibana.yml.j2 +++ b/ansible/roles/kibana/templates/kibana.yml.j2 @@ -2,7 +2,7 @@ kibana.defaultAppId: "{{ kibana_default_app_id }}" logging.dest: /var/log/kolla/kibana/kibana.log server.port: {{ kibana_server_port }} server.host: "{{ api_interface_address }}" -elasticsearch.url: "{{ elasticsearch_internal_endpoint }}" +elasticsearch.hosts: "{{ elasticsearch_internal_endpoint }}" elasticsearch.requestTimeout: {{ kibana_elasticsearch_request_timeout }} elasticsearch.shardTimeout: {{ kibana_elasticsearch_shard_timeout }} elasticsearch.ssl.verificationMode: "{{ 'full' if kibana_elasticsearch_ssl_verify | bool else 'none' }}" diff --git a/releasenotes/notes/upgrade-elk-to-7-release-c559d4c5dff05d2f.yaml b/releasenotes/notes/upgrade-elk-to-7-release-c559d4c5dff05d2f.yaml new file mode 100644 index 0000000000..9eb27b79f3 --- /dev/null +++ b/releasenotes/notes/upgrade-elk-to-7-release-c559d4c5dff05d2f.yaml @@ -0,0 +1,7 @@ +--- +upgrade: + - | + Update service configuration for the ELK 7 OSS release. A rolling upgrade + from ELK 6 is supported. Please see the official `upgrade notes + `__ + for more detail. diff --git a/tests/test-prometheus-efk.sh b/tests/test-prometheus-efk.sh index 1828ea6107..6fd6f214cb 100755 --- a/tests/test-prometheus-efk.sh +++ b/tests/test-prometheus-efk.sh @@ -8,8 +8,8 @@ set -o pipefail export PYTHONUNBUFFERED=1 function check_kibana { - # Query kibana, and check that the returned page looks like a kibana page. - KIBANA_URL=${OS_AUTH_URL%:*}:5601 + # Perform and validate a basic status page check + KIBANA_URL=${OS_AUTH_URL%:*}:5601/api/status output_path=$1 kibana_password=$(awk '$1 == "kibana_password:" { print $2 }' /etc/kolla/passwords.yml) args=( @@ -25,7 +25,25 @@ function check_kibana { if ! curl "${args[@]}" $KIBANA_URL > $output_path; then return 1 fi - if ! grep 'Kibana' $output_path >/dev/null; then + if ! grep 'Looking good' $output_path >/dev/null; then + return 1 + fi +} + +function check_elasticsearch { + # Verify that we see a healthy index created due to Fluentd forwarding logs + ELASTICSEARCH_URL=${OS_AUTH_URL%:*}:9200/_cluster/health + output_path=$1 + args=( + --include + --location + --fail + ) + if ! curl "${args[@]}" $ELASTICSEARCH_URL > $output_path; then + return 1 + fi + # NOTE(mgoddard): Status is yellow because no indices have been created. + if ! grep '"status":"yellow"' $output_path >/dev/null; then return 1 fi } @@ -75,7 +93,6 @@ function check_prometheus { } function test_kibana { - # TODO(mgoddard): Query elasticsearch for logs. echo "TESTING: Kibana" output_path=$(mktemp) attempt=1 @@ -92,6 +109,23 @@ function test_kibana { echo "SUCCESS: Kibana" } +function test_elasticsearch { + echo "TESTING: Elasticsearch" + output_path=$(mktemp) + attempt=1 + while ! check_elasticsearch $output_path; do + echo "Elasticsearch not accessible yet" + attempt=$((attempt+1)) + if [[ $attempt -eq 12 ]]; then + echo "FAILED: Elasticsearch did not become accessible. Response:" + cat $output_path + return 1 + fi + sleep 10 + done + echo "SUCCESS: Elasticsearch" +} + function test_grafana { echo "TESTING: Grafana" output_path=$(mktemp) @@ -131,6 +165,7 @@ function test_prometheus_efk_logged { . /etc/kolla/admin-openrc.sh test_kibana + test_elasticsearch test_grafana test_prometheus }