Upgrade service configuration for ELK 7

ELK 7 requires some minor changes from the existing ELK 6 config.

Depends-On: Icfa3db5788b25f70ee75411dbaf20d8d4a6a734b
Change-Id: I9815d202a77da0477aea43d714a5def8a24724fa

ansible/roles/elasticsearch/templates/elasticsearch.yml.j2

@@ -5,9 +5,10 @@ node.name: "{{ 'api' | kolla_address | put_address_in_context('url') }}"
 network.host: "{{ 'api' | kolla_address | put_address_in_context('url') }}"
 
 cluster.name: "{{ elasticsearch_cluster_name }}"
+cluster.initial_master_nodes: [{% for host in groups['elasticsearch'] %}"{{ 'api' | kolla_address(host) }}"{% if not loop.last %},{% endif %}{% endfor %}]
 node.master: true
 node.data: true
-discovery.zen.ping.unicast.hosts: [{% for host in groups['elasticsearch'] %}"{{ 'api' | kolla_address(host) | put_address_in_context('url') }}"{% if not loop.last %},{% endif %}{% endfor %}]
+discovery.seed_hosts: [{% for host in groups['elasticsearch'] %}"{{ 'api' | kolla_address(host) | put_address_in_context('url') }}"{% if not loop.last %},{% endif %}{% endfor %}]
 
 discovery.zen.minimum_master_nodes: {{ minimum_master_nodes }}
 http.port: {{ elasticsearch_port }}

ansible/roles/kibana/templates/kibana.json.j2

@@ -1,5 +1,5 @@
 {
-    "command": "/usr/share/kibana/bin/kibana",
+    "command": "/usr/share/kibana/bin/kibana --config /etc/kibana/kibana.yml",
     "config_files": [
         {
             "source": "{{ container_config_directory }}/kibana.yml",

ansible/roles/kibana/templates/kibana.yml.j2

@@ -2,7 +2,7 @@ kibana.defaultAppId: "{{ kibana_default_app_id }}"
 logging.dest: /var/log/kolla/kibana/kibana.log
 server.port: {{ kibana_server_port }}
 server.host: "{{ api_interface_address }}"
-elasticsearch.url: "{{ elasticsearch_internal_endpoint }}"
+elasticsearch.hosts: "{{ elasticsearch_internal_endpoint }}"
 elasticsearch.requestTimeout: {{ kibana_elasticsearch_request_timeout }}
 elasticsearch.shardTimeout: {{ kibana_elasticsearch_shard_timeout }}
 elasticsearch.ssl.verificationMode: "{{ 'full' if kibana_elasticsearch_ssl_verify | bool else 'none' }}"

releasenotes/notes/upgrade-elk-to-7-release-c559d4c5dff05d2f.yaml

@@ -0,0 +1,7 @@
+---
+upgrade:
+  - |
+    Update service configuration for the ELK 7 OSS release. A rolling upgrade
+    from ELK 6 is supported. Please see the official `upgrade notes
+    <https://www.elastic.co/guide/en/elastic-stack/current/upgrading-elastic-stack.html>`__
+    for more detail.

tests/test-prometheus-efk.sh

@@ -8,8 +8,8 @@ set -o pipefail
 export PYTHONUNBUFFERED=1
 
 function check_kibana {
-    # Query kibana, and check that the returned page looks like a kibana page.
-    KIBANA_URL=${OS_AUTH_URL%:*}:5601
+    # Perform and validate a basic status page check
+    KIBANA_URL=${OS_AUTH_URL%:*}:5601/api/status
     output_path=$1
     kibana_password=$(awk '$1 == "kibana_password:" { print $2 }' /etc/kolla/passwords.yml)
     args=(
@@ -25,7 +25,25 @@ function check_kibana {
     if ! curl "${args[@]}" $KIBANA_URL > $output_path; then
         return 1
     fi
-    if ! grep '<title>Kibana</title>' $output_path >/dev/null; then
+    if ! grep 'Looking good' $output_path >/dev/null; then
+        return 1
+    fi
+}
+
+function check_elasticsearch {
+    # Verify that we see a healthy index created due to Fluentd forwarding logs
+    ELASTICSEARCH_URL=${OS_AUTH_URL%:*}:9200/_cluster/health
+    output_path=$1
+    args=(
+        --include
+        --location
+        --fail
+    )
+    if ! curl "${args[@]}" $ELASTICSEARCH_URL > $output_path; then
+        return 1
+    fi
+    # NOTE(mgoddard): Status is yellow because no indices have been created.
+    if ! grep '"status":"yellow"' $output_path >/dev/null; then
         return 1
     fi
 }
@@ -75,7 +93,6 @@ function check_prometheus {
 }
 
 function test_kibana {
-    # TODO(mgoddard): Query elasticsearch for logs.
     echo "TESTING: Kibana"
     output_path=$(mktemp)
     attempt=1
@@ -92,6 +109,23 @@ function test_kibana {
     echo "SUCCESS: Kibana"
 }
 
+function test_elasticsearch {
+    echo "TESTING: Elasticsearch"
+    output_path=$(mktemp)
+    attempt=1
+    while ! check_elasticsearch $output_path; do
+        echo "Elasticsearch not accessible yet"
+        attempt=$((attempt+1))
+        if [[ $attempt -eq 12 ]]; then
+            echo "FAILED: Elasticsearch did not become accessible. Response:"
+            cat $output_path
+            return 1
+        fi
+        sleep 10
+    done
+    echo "SUCCESS: Elasticsearch"
+}
+
 function test_grafana {
     echo "TESTING: Grafana"
     output_path=$(mktemp)
@@ -131,6 +165,7 @@ function test_prometheus_efk_logged {
     . /etc/kolla/admin-openrc.sh
 
     test_kibana
+    test_elasticsearch
     test_grafana
     test_prometheus
 }