Merge "Upgrade to Logstash 6"

docker/base/Dockerfile.j2

@@ -90,13 +90,10 @@ COPY dnf.conf /etc/dnf/dnf.conf
     ] %}
 {% elif base_arch == 'aarch64' %}
     {% set base_yum_repo_files = [
+        'elasticsearch.repo',
         'grafana.repo',
         'rabbitmq_rabbitmq-server.repo',
     ] %}
-    # FIXME(mgoddard): Not available for CentOS 8 yet.
-    #
-    #   'elasticsearch.repo',
-    #
 
     {% set base_yum_repo_keys = [
         'https://packages.grafana.com/gpg.key',

docker/logstash/Dockerfile.j2

@@ -9,58 +9,26 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
 {{ macros.configure_user(name='logstash', shell='/bin/bash', homedir='/usr/share/logstash') }}
 
+{{ macros.enable_extra_repos(['logstash']) }}
+
 {% if base_package_type == 'rpm' %}
-    {% set logstash_packages = [
+    {% set elk_java_package = 'java-11-openjdk-headless' %}
-        'java-1.8.0-openjdk-headless'
+ENV JAVA_HOME /usr/lib/jvm/jre-11-openjdk/
-    ] %}
-
-ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0-openjdk/
-
 {% elif base_package_type == 'deb' %}
-
+    {% set elk_java_package = 'openjdk-11-jre-headless' %}
-    {% if base_distro == 'debian' %}
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-{{debian_arch}}/
-        {% set java_version = '11' %}
-    {% elif base_distro == 'ubuntu' %}
-        {% set java_version = '8' %}
-    {% endif %}
-
-    {% set logstash_packages = [
-        'openjdk-' + java_version + '-jre-headless',
-        'logrotate'
-    ] %}
-
-ENV JAVA_HOME /usr/lib/jvm/java-{{java_version}}-openjdk-{{debian_arch}}/
-
 {% endif %}
 
+{% set logstash_packages = [
+    'logstash-oss',
+] %}
+
+# Post-install script of Logstash fails when trying to install Logstash and
+# Java together.
+{{ macros.install_packages([elk_java_package]) }}
 {{ macros.install_packages(logstash_packages | customizable("packages")) }}
 COPY extend_start.sh /usr/local/bin/kolla_extend_start
 
-# FIXME(dszumski): Logstash was added to Kolla for Monasca. Monasca is
-# currently tied to an old messaging version in Kafka (0.9) and doesn't
-# work with Logstash 5.x or above. When Monasca is updated to use a more
-# recent messaging version this patch should be reverted and the Monasca
-# Logstash configuration files upgraded to use the new format.
-{% if base_package_type == 'rpm' %}
-
-ENV logstash_rpm_url=https://download.elastic.co/logstash/logstash/packages/centos/logstash-2.4.1.noarch.rpm
-ENV logstash_rpm_sha1sum=68c5e78c3d941f06f77d638c0756b6aea715bec5
-RUN curl -sSL -o /tmp/logstash.rpm ${logstash_rpm_url} \
-    && echo "${logstash_rpm_sha1sum} /tmp/logstash.rpm" | sha1sum -c \
-    && dnf -y install /tmp/logstash.rpm \
-    && rm -f /tmp/logstash.rpm
-
-{% elif base_package_type == 'deb' %}
-
-ENV logstash_deb_url=https://download.elastic.co/logstash/logstash/packages/debian/logstash-2.4.1_all.deb
-ENV logstash_deb_sha1sum=7ba3b174a3ef48a7d0945d9b5c7f12c5005abb47
-RUN curl -sSL -o /tmp/logstash.deb ${logstash_deb_url} \
-    && echo "${logstash_deb_sha1sum} /tmp/logstash.deb" | sha1sum -c \
-    && apt install /tmp/logstash.deb \
-    && rm -f /tmp/logstash.deb
-
-{% endif %}
-
 RUN chmod 755 /usr/local/bin/kolla_extend_start
 
 {% block logstash_footer %}{% endblock %}

kolla/template/repos.yaml

@@ -10,6 +10,7 @@ centos:
   grafana: "grafana"
   hacluster: "HighAvailability"
   influxdb: "influxdb"
+  logstash: "elasticsearch-kibana-logstash-6.x"
   kibana: "elasticsearch-kibana-logstash-6.x"
   opstools: "centos-opstools"
   rabbitmq: "rabbitmq_rabbitmq-server"
@@ -24,6 +25,7 @@ centos-aarch64:
   extras: "extras"
   grafana: "grafana"
   hacluster: "HighAvailability"
+  logstash: "elasticsearch-kibana-logstash-6.x"
   opstools: "centos-opstools"
   rabbitmq: "rabbitmq_rabbitmq-server"
 
@@ -34,6 +36,7 @@ centos-ppc64le:
   epel-modular: "epel-modular"
   extras: "extras"
   hacluster: "HighAvailability"
+  logstash: "elasticsearch-kibana-logstash-6.x"
   opstools: "centos-opstools"
   rabbitmq: "rabbitmq_rabbitmq-server"
 
@@ -48,6 +51,7 @@ debian:
   erlang: "deb https://dl.bintray.com/rabbitmq-erlang/debian/ buster erlang"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/debian buster stable"
+  logstash: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ buster main"
   td-agent: "deb http://packages.treasuredata.com/3/debian/buster buster contrib"
@@ -57,6 +61,7 @@ debian-aarch64:
   erlang: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/debian buster stable"
+  logstash: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   libvirt: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./"
   rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ buster main"
@@ -66,6 +71,7 @@ ubuntu:
   erlang: "deb https://dl.bintray.com/rabbitmq-erlang/debian/ bionic erlang"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/ubuntu bionic stable"
+  logstash: "deb https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   kibana: "deb https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main"
   qdrouterd: "deb http://ppa.launchpad.net/qpid/released/ubuntu/ bionic main"
@@ -77,6 +83,7 @@ ubuntu-aarch64:
   erlang: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/ubuntu-bionic ./"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/ubuntu bionic stable"
+  logstash: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main"
   rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ bionic main"
 

releasenotes/notes/add-logstash6-everywhere-7f9027ff17c6022d.yaml

@@ -0,0 +1,4 @@
+---
+upgrade:
+  - |
+    The Logstash image has been upgraded from Logstash 2 to Logstash 6.