Upgrade Elasticsearch and Kibana to 6.x

Start using oss tagged binaries. These binaries contain only features
that are available under the Apache 2.0 license.

Implements: blueprint elasticsearch-kibana-version-upgrade

Co-authored-by: Dincer Celik <hello@dincercelik.com>
Change-Id: I8b52b2630a72393bf4b9c7095fc67db6c36c27fa

docker/base/apt_preferences.debian

@@ -24,12 +24,12 @@ Pin: release o=obs://private/home:marcin.juszkiewicz/debian-buster
 Pin-Priority: 100
 
 # Unless it is kibana (on aarch64)
-Package: kibana
+Package: kibana kibana-oss
 Pin: release o=obs://private/home:marcin.juszkiewicz/debian-buster
 Pin-Priority: 600
 
 # on x86-64 use upstream package for kibana
-Package: kibana
+Package: kibana kibana-oss
 Pin: release o=elastic,b=amd64
 Pin-Priority: 700
 

docker/base/elasticsearch.repo

@@ -1,8 +1,6 @@
-[elasticsearch-kibana-logstash-5.x]
+[elasticsearch-kibana-logstash-6.x]
-name=ELK repository for 5.x packages
+name=ELK repository for 6.x packages
-baseurl=https://artifacts.elastic.co/packages/5.x/yum
+baseurl=https://artifacts.elastic.co/packages/oss-6.x/yum
 gpgcheck=1
 gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
 enabled=0
-autorefresh=1
-type=rpm-md

docker/elasticsearch/elasticsearch/Dockerfile.j2

@@ -13,19 +13,22 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
 {% if base_package_type == 'rpm' %}
     {% set elasticsearch_packages = [
-        'elasticsearch',
+        'java-11-openjdk-headless',
-        'java-1.8.0-openjdk-headless',
+        'elasticsearch-oss',
     ] %}
 
-ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0-openjdk/
+ENV JAVA_HOME /usr/lib/jvm/jre-11-openjdk/
 
 {% elif base_package_type == 'deb' %}
     {% set elasticsearch_packages = [
-        'default-jre-headless',
+        'elasticsearch-oss',
-        'elasticsearch',
     ] %}
 
-ENV JAVA_HOME /usr/lib/jvm/java-{{java_version}}-openjdk-{{debian_arch}}/
+# NOTE(osmanlicilegi): post-install script of elasticsearch fails when trying to
+# install elasticsearch and java together.
+{{ macros.install_packages(['default-jre-headless']) }}
+
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-{{debian_arch}}/
 
 {% endif %}
 

docker/kibana/Dockerfile.j2

@@ -11,8 +11,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
 {{ macros.configure_user(name='kibana') }}
 
-{% set kibana_packages = ['kibana'] %}
+{% set kibana_packages = ['kibana-oss'] %}
-
 {{ macros.install_packages(kibana_packages | customizable("packages")) }}
 
 COPY extend_start.sh /usr/local/bin/kolla_extend_start

kolla/template/repos.yaml

@@ -2,14 +2,14 @@
 # TODO(mgoddard): Remove this when CentOS 7 support is no longer present.
 centos:
   ceph: "centos-ceph-nautilus"
-  elasticsearch: "elasticsearch-kibana-logstash-5.x"
+  elasticsearch: "elasticsearch-kibana-logstash-6.x"
   epel: "epel"
   erlang: "rabbitmq_rabbitmq-erlang"
   extras: "extras"
   grafana: "grafana"
   hacluster: "network_ha-clustering_Stable"
   influxdb: "influxdb"
-  kibana: "elasticsearch-kibana-logstash-5.x"
+  kibana: "elasticsearch-kibana-logstash-6.x"
   nfs-ganesha: "centos-nfs-ganesha28"
   opendaylight: "opendaylight"
   opstools: "centos-release-opstools"
@@ -19,7 +19,7 @@ centos:
 # TODO(mgoddard): Remove this when CentOS 7 support is no longer present.
 centos-aarch64:
   ceph: "centos-ceph-nautilus"
-  elasticsearch: "elasticsearch-kibana-logstash-5.x"
+  elasticsearch: "elasticsearch-kibana-logstash-6.x"
   epel: "epel"
   erlang: "rabbitmq_rabbitmq-erlang"
   extras: "extras"
@@ -32,7 +32,7 @@ centos-aarch64:
 # TODO(mgoddard): Remove this when CentOS 7 support is no longer present.
 centos-ppc64le:
   ceph: "centos-ceph-nautilus"
-  elasticsearch: "elasticsearch-kibana-logstash-5.x"
+  elasticsearch: "elasticsearch-kibana-logstash-6.x"
   epel: "epel"
   extras: "extras"
   nfs-ganesha: "centos-nfs-ganesha28"
@@ -56,16 +56,16 @@ centos8:
 rhel: {}
 
 debian:
-  elasticsearch: "deb [arch=amd64] https://artifacts.elastic.co/packages/5.x/apt stable main"
+  elasticsearch: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   erlang: "deb https://dl.bintray.com/rabbitmq-erlang/debian/ buster erlang"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/debian buster stable"
-  kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/5.x/apt stable main"
+  kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ buster main"
   td-agent: "deb http://packages.treasuredata.com/3/debian/buster buster contrib"
 
 debian-aarch64:
-  elasticsearch: "deb [arch=amd64] https://artifacts.elastic.co/packages/5.x/apt stable main"
+  elasticsearch: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   erlang: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/debian buster stable"
@@ -74,11 +74,11 @@ debian-aarch64:
   rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ buster main"
 
 ubuntu:
-  elasticsearch: "deb https://artifacts.elastic.co/packages/5.x/apt stable main"
+  elasticsearch: "deb https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   erlang: "deb https://dl.bintray.com/rabbitmq-erlang/debian/ bionic erlang"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/ubuntu bionic stable"
-  kibana: "deb https://artifacts.elastic.co/packages/5.x/apt stable main"
+  kibana: "deb https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main"
   opendaylight: "deb http://ppa.launchpad.net/odl-team/carbon/ubuntu xenial main"
   qdrouterd: "deb http://ppa.launchpad.net/qpid/released/ubuntu/ bionic main"
@@ -87,7 +87,7 @@ ubuntu:
   td-agent: "deb http://packages.treasuredata.com/3/ubuntu/bionic/ bionic contrib"
 
 ubuntu-aarch64:
-  elasticsearch: "deb [arch=amd64] https://artifacts.elastic.co/packages/5.x/apt stable main"
+  elasticsearch: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   erlang: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/ubuntu-bionic ./"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/ubuntu bionic stable"

kolla/tests/test_methods.py

@@ -109,8 +109,8 @@ class MethodsTest(base.TestCase):
         expectCmd = 'RUN echo "deb https://packages.grafana.com/oss/deb '
         expectCmd += 'stable main" >/etc/apt/sources.list.d/grafana.list && '
         expectCmd += 'echo "deb [arch=amd64] '
-        expectCmd += 'https://artifacts.elastic.co/packages/5.x/apt stable '
+        expectCmd += 'https://artifacts.elastic.co/packages/oss-6.x/apt '
-        expectCmd += 'main" >/etc/apt/sources.list.d/kibana.list'
+        expectCmd += 'stable main" >/etc/apt/sources.list.d/kibana.list'
         self.assertEqual(expectCmd, result)
 
     def test_disable_repos_centos(self):

releasenotes/notes/upgrade-elasticsearch-and-kibana-to-6.X-ee448b2e8ae51922.yaml

@@ -0,0 +1,6 @@
+---
+upgrade:
+  - |
+    Upgrades Elasticsearch and Kibana to 6.x. See `blueprint
+    <https://blueprints.launchpad.net/kolla/+spec/elasticsearch-kibana-version-upgrade>`__
+    for details.