Merge "Drop elasticsearch, kibana and logstash"

This commit is contained in:
Zuul 2022-12-07 13:30:25 +00:00 committed by Gerrit Code Review
commit d5886bc3b8
21 changed files with 18 additions and 337 deletions

View File

@ -62,10 +62,6 @@ COPY dnf.conf /etc/dnf/dnf.conf
{% endblock %}
#### BEGIN REPO ENABLEMENT
{#
Repo files not available for CentOS Stream 9:
'elasticsearch.repo', # signed with SHA1 key https://github.com/elastic/elasticsearch/issues/85876
#}
{% set base_yum_repo_files = [
'grafana.repo',
'influxdb.repo',
@ -80,7 +76,6 @@ Repo files not available for CentOS Stream 9:
{#
SHA1 keys are not supported in RHEL9: https://github.com/rpm-software-management/rpm/issues/1977
'https://artifacts.elastic.co/GPG-KEY-elasticsearch',
'https://packages.grafana.com/gpg.key',
#}
{% set base_yum_repo_keys = [
@ -320,7 +315,6 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
{# NOTE(hrw): type field defaults to 'asc' which is used for single keys #}
{% set base_remote_apt_keys = [
{'name': 'elasticsearch', 'url': 'https://artifacts.elastic.co/GPG-KEY-elasticsearch'},
{'name': 'grafana', 'url': 'https://packages.grafana.com/gpg.key'},
{'name': 'influxdb', 'url': 'https://repos.influxdata.com/influxdb.key'},
{'name': 'mariadb', 'url': 'https://downloads.mariadb.com/MariaDB/mariadb-keyring-2019.gpg', 'type': 'gpg'},

View File

@ -6,12 +6,6 @@ Package: erlang*
Pin: version 1:25.*
Pin-Priority: 1000
# NOTE(mgoddard): logstash 7.9.x is the last version that supports
# Elasticsearch OSS.
Package: logstash-oss
Pin: version 7.9.*
Pin-Priority: 1000
# NOTE(hrw): we do not want backports unless requested
Package: *
Pin: release n=bullseye-backports

View File

@ -6,12 +6,6 @@ Package: erlang*
Pin: version 1:25.*
Pin-Priority: 1000
# NOTE(mgoddard): logstash 7.9.x is the last version that supports
# Elasticsearch OSS.
Package: logstash-oss
Pin: version 7.9.*
Pin-Priority: 1000
Package: *
Pin: origin dlm.mariadb.com
Pin-Priority: 1000

View File

@ -1,6 +0,0 @@
[elasticsearch-kibana-logstash-7.x]
name=ELK repository for 7.x packages
baseurl=https://artifacts.elastic.co/packages/oss-7.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=0

View File

@ -31,7 +31,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
'collectd-disk',
'collectd-dns',
'collectd-ipmi',
'collectd-log_logstash',
'collectd-logparser',
'collectd-mcelog',
'collectd-mysql',

View File

@ -1,44 +0,0 @@
FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }}
{% block labels %}
LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
{% endblock %}
{% block elasticsearch_curator_header %}{% endblock %}
{% import "macros.j2" as macros with context %}
{% if base_package_type == 'rpm' %}
{% set elasticsearch_curator_packages = [
'cronie',
'python3-wheel'
] %}
{% elif base_package_type == 'deb' %}
{% set elasticsearch_curator_packages = [
'cron',
'python3-setuptools',
'python3-wheel'
] %}
{% endif %}
{{ macros.install_packages(elasticsearch_curator_packages | customizable("packages")) }}
{{ macros.configure_user(name='elasticsearch') }}
# NOTE(parallax): pin elasticsearch due to bug: https://bugs.launchpad.net/kolla/+bug/1941073
{% set elasticsearch_curator_pip_packages = [
'elasticsearch==7.13.*',
'elasticsearch-curator'
] %}
COPY extend_start.sh /usr/local/bin/kolla_extend_start
RUN {{ macros.install_pip(['pip', 'wheel', 'setuptools'], constraints=false) }} \
&& {{ macros.install_pip(elasticsearch_curator_pip_packages | customizable("pip_packages"), constraints=false) }} \
&& mkdir -p /etc/elasticsearch-curator \
&& chmod 644 /usr/local/bin/kolla_extend_start
{% block elasticsearch_curator_base_footer %}{% endblock %}
{% block footer %}{% endblock %}
# NOTE(dszumski): Cron runs as root but should be configured to launch Curator
# by the elasticsearch user.

View File

@ -1,8 +0,0 @@
#!/bin/bash
# NOTE(dszumski): Cron runs as root but should be configured to launch Curator
# by the elasticsearch user. Therefore this directory needs to be owned by
# the elasticsearch user.
if [[ ! -d "/var/log/kolla/elasticsearch" ]]; then
install -d -m 0755 -o elasticsearch -g elasticsearch /var/log/kolla/elasticsearch
fi

View File

@ -1,54 +0,0 @@
FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }}
{% block labels %}
LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
{% endblock %}
{% block elasticsearch_header %}{% endblock %}
{% import "macros.j2" as macros with context %}
{{ macros.configure_user(name='elasticsearch', shell='/bin/bash', homedir='/usr/share/elasticsearch') }}
{{ macros.enable_extra_repos(['elasticsearch']) }}
{% if base_package_type == 'rpm' %}
# NOTE(hrw): post-install script of elasticsearch fails when trying to
# install elasticsearch and java together.
{{ macros.install_packages(['java-11-openjdk-headless']) }}
{% set elasticsearch_packages = [
'elasticsearch-oss',
] %}
ENV JAVA_HOME /usr/lib/jvm/jre-11-openjdk/
{% elif base_package_type == 'deb' %}
{% set elasticsearch_packages = [
'elasticsearch-oss',
] %}
# NOTE(osmanlicilegi): post-install script of elasticsearch fails when trying to
# install elasticsearch and java together.
{{ macros.install_packages(['default-jre-headless']) }}
ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-{{debian_arch}}/
{% endif %}
ENV ES_SKIP_SET_KERNEL_PARAMETERS true
{{ macros.install_packages(elasticsearch_packages | customizable("packages")) }}
COPY extend_start.sh /usr/local/bin/kolla_extend_start
# NOTE: By default the shell of the elasticsearch user is /bin/false. We have to
# change the shell to /bin/bash to be able to run elasticsearch as a non-root user.
#
# https://discuss.elastic.co/t/running-as-non-root-user-service-wrapper-has-changed/7863
RUN chmod 644 /usr/local/bin/kolla_extend_start
{% block elasticsearch_footer %}{% endblock %}
{% block footer %}{% endblock %}
USER elasticsearch

View File

@ -1,8 +0,0 @@
#!/bin/bash
if [[ ! -d "/var/log/kolla/elasticsearch" ]]; then
mkdir -p /var/log/kolla/elasticsearch
fi
if [[ $(stat -c %a /var/log/kolla/elasticsearch) != "755" ]]; then
chmod 755 /var/log/kolla/elasticsearch
fi

View File

@ -34,12 +34,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
COPY fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers
COPY extend_start.sh /usr/local/bin/kolla_extend_start
# NOTE(frickler): Downgrading elasticsearch gems for compatibility with OSS versions of ES
{{ macros.install_packages(fluentd_packages | customizable("packages")) }} \
&& td-agent-gem install elasticsearch:7.13.0 \
&& td-agent-gem uninstall "elasticsearch:>7.13.0" \
&& td-agent-gem uninstall "elasticsearch-api:>7.13.0" \
&& td-agent-gem uninstall "elasticsearch-transport:>7.13.0" \
&& chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \
&& mkdir -p /etc/{{ fluentd_user }} \
&& mkdir -p /var/run/{{ fluentd_user }} \
@ -49,7 +44,6 @@ COPY extend_start.sh /usr/local/bin/kolla_extend_start
{% block fluentd_plugins_install %}
{% set fluentd_plugins = [
'fluent-plugin-elasticsearch',
'fluent-plugin-grok-parser',
'fluent-plugin-prometheus',
'fluent-plugin-rewrite-tag-filter',

View File

@ -1,37 +0,0 @@
FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }}
{% block labels %}
LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
{% endblock %}
{% block kibana_header %}{% endblock %}
{% import "macros.j2" as macros with context %}
{{ macros.enable_extra_repos(['kibana']) }}
{{ macros.configure_user(name='kibana') }}
# NOTE(hrw): upstream package ships x86_64 binaries for NodeJS. On other
# architectures we install NodeJS from distro and remove binaries.
{% if base_arch != 'x86_64' and base_distro in ['debian'] %}
RUN dpkg --add-architecture amd64 && \
{{ macros.install_packages(['nodejs', 'kibana-oss:amd64'], chain=True) }} \
&& rm -rf /usr/share/kibana/node
{% else %}
{% set kibana_packages = ['kibana-oss'] %}
{{ macros.install_packages(kibana_packages | customizable("packages")) }}
{% endif %}
COPY extend_start.sh /usr/local/bin/kolla_extend_start
RUN chmod 644 /usr/local/bin/kolla_extend_start
{% block kibana_footer %}{% endblock %}
{% block footer %}{% endblock %}
USER kibana

View File

@ -1,10 +0,0 @@
#!/bin/bash
KIBANA_LOG_DIR="/var/log/kolla/kibana"
if [[ ! -d "${KIBANA_LOG_DIR}" ]]; then
mkdir -p "${KIBANA_LOG_DIR}"
fi
if [[ $(stat -c %U:%G "${KIBANA_LOG_DIR}") != "kibana:kolla" ]]; then
chown kibana:kolla "${KIBANA_LOG_DIR}"
fi

View File

@ -1,45 +0,0 @@
FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }}
{% block labels %}
LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
{% endblock %}
{% block logstash_header %}{% endblock %}
{% import "macros.j2" as macros with context %}
{{ macros.configure_user(name='logstash', shell='/bin/bash', homedir='/usr/share/logstash') }}
{{ macros.enable_extra_repos(['logstash']) }}
{% if base_package_type == 'rpm' %}
{% set elk_java_package = 'java-11-openjdk-headless' %}
ENV JAVA_HOME /usr/lib/jvm/jre-11-openjdk/
{% elif base_package_type == 'deb' %}
{% set elk_java_package = 'openjdk-11-jre-headless' %}
ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-{{debian_arch}}/
{% endif %}
# NOTE(mgoddard): logstash 7.9.x is the last version that supports
# Elasticsearch OSS.
{% if base_package_type == 'rpm' %}
{% set logstash_packages = [
'logstash-oss-7.9.*',
] %}
{% else %}
{% set logstash_packages = [
'logstash-oss',
] %}
{% endif %}
# Post-install script of Logstash fails when trying to install Logstash and
# Java together.
{{ macros.install_packages([elk_java_package]) }}
{{ macros.install_packages(logstash_packages | customizable("packages")) }}
COPY extend_start.sh /usr/local/bin/kolla_extend_start
RUN chmod 644 /usr/local/bin/kolla_extend_start
{% block logstash_footer %}{% endblock %}
{% block footer %}{% endblock %}
USER logstash

View File

@ -1,8 +0,0 @@
#!/bin/bash
if [[ ! -d "/var/log/kolla/logstash" ]]; then
mkdir -p /var/log/kolla/logstash
fi
if [[ $(stat -c %a /var/log/kolla/logstash) != "755" ]]; then
chmod 755 /var/log/kolla/logstash
fi

View File

@ -85,7 +85,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
'cryptography',
'debtcollector',
'decorator',
'elasticsearch',
'eventlet',
'fasteners',
'fluent-logger',

View File

@ -52,16 +52,13 @@ _PROFILE_OPTS = [
cfg.ListOpt('infra',
default=[
'cron',
'elasticsearch',
'etcd',
'fluentd',
'haproxy',
'hacluster',
'keepalived',
'kibana',
'kolla-toolbox',
'letsencrypt',
'logstash',
'mariadb',
'memcached',
'opensearch',

View File

@ -57,7 +57,7 @@ USERS = {
'uid': 42411,
'gid': 42411,
},
'elasticsearch-user': {
'elasticsearch-user': { # unused user (elasticsearch dropped)
'uid': 42412,
'gid': 42412,
},
@ -105,7 +105,7 @@ USERS = {
'uid': 42425,
'gid': 42425,
},
'kibana-user': {
'kibana-user': { # unused user (kibana dropped)
'uid': 42426,
'gid': 42426,
},
@ -285,7 +285,7 @@ USERS = {
'uid': 42477, # unused user, but we need the group for vhost socket
'gid': 42477,
},
'logstash-user': {
'logstash-user': { # unused user (elasticsearch dropped)
'uid': 42478,
'gid': 42478,
},

View File

@ -23,13 +23,9 @@ UNBUILDABLE_IMAGES = {
},
# Issues for SHA1 keys:
# https://github.com/elastic/elasticsearch/issues/85876
# https://github.com/grafana/grafana/issues/41036
'centos': {
"elasticsearch", # SHA1 gpg key
"hacluster-pcs", # Missing crmsh package
"kibana", # SHA1 gpg key
"logstash", # SHA1 gpg key
"nova-spicehtml5proxy", # Missing spicehtml5 package
"ovsdpdk", # Not supported on CentOS
"tgtd", # Not supported on CentOS
@ -39,10 +35,7 @@ UNBUILDABLE_IMAGES = {
},
'rocky': {
"elasticsearch", # SHA1 gpg key
"hacluster-pcs", # Missing crmsh package
"kibana", # SHA1 gpg key
"logstash", # SHA1 gpg key
"nova-spicehtml5proxy", # Missing spicehtml5 package
"ovsdpdk", # Not supported on CentOS
"tgtd", # Not supported on CentOS
@ -55,11 +48,9 @@ UNBUILDABLE_IMAGES = {
'ubuntu+aarch64': {
"barbican-base", # https://github.com/unbit/uwsgi/issues/2434
"kibana", # no binary package
},
'centos+aarch64': {
"kibana", # no binary package
"telegraf", # no binary package
},
}

View File

@ -2,15 +2,12 @@
centos:
ceph: "centos-ceph-quincy"
crb: "crb"
elasticsearch: "elasticsearch-kibana-logstash-7.x"
epel: "epel"
erlang: "rabbitmq_rabbitmq-erlang"
extras: "extras"
grafana: "grafana"
hacluster: "highavailability"
influxdb: "influxdb"
kibana: "elasticsearch-kibana-logstash-7.x"
logstash: "elasticsearch-kibana-logstash-7.x"
mariadb: "mariadb"
openvswitch: "centos-nfv-openvswitch"
opstools: "centos-opstools"
@ -21,15 +18,12 @@ centos:
centos-aarch64:
ceph: "centos-ceph-quincy"
crb: "crb"
elasticsearch: "elasticsearch-kibana-logstash-7.x"
epel: "epel"
erlang: "copr-hrw-erlang-for-rabbitmq"
extras: "extras"
grafana: "grafana"
hacluster: "highavailability"
influxdb: "influxdb"
kibana: "elasticsearch-kibana-logstash-7.x"
logstash: "elasticsearch-kibana-logstash-7.x"
mariadb: "mariadb"
openvswitch: "centos-nfv-openvswitch"
opstools: "centos-opstools"
@ -38,11 +32,6 @@ centos-aarch64:
td-agent: "treasuredata"
debian:
elasticsearch:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
erlang:
url: "https://ppa.launchpadcontent.net/rabbitmq/rabbitmq-erlang/ubuntu"
suite: "focal"
@ -58,16 +47,6 @@ debian:
suite: "bullseye"
component: "stable"
gpg_key: "influxdb.asc"
logstash:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
kibana:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
mariadb:
url: "https://dlm.mariadb.com/repo/mariadb-server/10.6/repo/debian"
suite: "bullseye"
@ -90,11 +69,6 @@ debian:
gpg_key: "proxysql.asc"
debian-aarch64:
elasticsearch:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
erlang:
url: "https://ppa.launchpadcontent.net/rabbitmq/rabbitmq-erlang/ubuntu"
suite: "focal"
@ -110,16 +84,6 @@ debian-aarch64:
suite: "bullseye"
component: "stable"
gpg_key: "influxdb.asc"
logstash:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
kibana:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
mariadb:
url: "https://dlm.mariadb.com/repo/mariadb-server/10.6/repo/debian"
suite: "bullseye"
@ -144,15 +108,12 @@ debian-aarch64:
rocky:
ceph: "centos-ceph-quincy"
crb: "crb"
elasticsearch: "elasticsearch-kibana-logstash-7.x"
epel: "epel"
erlang: "rabbitmq_rabbitmq-erlang"
extras: "extras"
grafana: "grafana"
hacluster: "highavailability"
influxdb: "influxdb"
kibana: "elasticsearch-kibana-logstash-7.x"
logstash: "elasticsearch-kibana-logstash-7.x"
mariadb: "mariadb"
openvswitch: "centos-nfv-openvswitch"
opstools: "centos-opstools"
@ -163,15 +124,12 @@ rocky:
rocky-aarch64:
ceph: "centos-ceph-quincy"
crb: "crb"
elasticsearch: "elasticsearch-kibana-logstash-7.x"
epel: "epel"
erlang: "copr-hrw-erlang-for-rabbitmq"
extras: "extras"
grafana: "grafana"
influxdb: "influxdb"
hacluster: "highavailability"
kibana: "elasticsearch-kibana-logstash-7.x"
logstash: "elasticsearch-kibana-logstash-7.x"
mariadb: "mariadb"
openvswitch: "centos-nfv-openvswitch"
opstools: "centos-opstools"
@ -180,11 +138,6 @@ rocky-aarch64:
td-agent: "treasuredata"
ubuntu:
elasticsearch:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
erlang:
url: "https://ppa.launchpadcontent.net/rabbitmq/rabbitmq-erlang/ubuntu"
suite: "jammy"
@ -200,16 +153,6 @@ ubuntu:
suite: "jammy"
component: "stable"
gpg_key: "influxdb.asc"
logstash:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
kibana:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
mariadb:
url: "https://dlm.mariadb.com/repo/mariadb-server/10.6/repo/ubuntu"
suite: "jammy"
@ -232,11 +175,6 @@ ubuntu:
gpg_key: "proxysql.asc"
ubuntu-aarch64:
elasticsearch:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
erlang:
url: "https://ppa.launchpadcontent.net/rabbitmq/rabbitmq-erlang/ubuntu"
suite: "jammy"
@ -252,11 +190,6 @@ ubuntu-aarch64:
suite: "jammy"
component: "stable"
gpg_key: "influxdb.asc"
logstash:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
mariadb:
url: "https://dlm.mariadb.com/repo/mariadb-server/10.6/repo/ubuntu"
suite: "jammy"

View File

@ -97,7 +97,7 @@ class MethodsTest(base.TestCase):
'base_package_type': 'deb'
}
result = methods.handle_repos(template_vars, ['grafana', 'kibana'],
result = methods.handle_repos(template_vars, ['grafana', 'rabbitmq'],
'enable')
expectCmd = "RUN echo 'Uris: https://packages.grafana.com/oss/deb' "
expectCmd += ">/etc/apt/sources.list.d/grafana.sources && "
@ -111,16 +111,16 @@ class MethodsTest(base.TestCase):
expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
expectCmd += "echo 'Uris: "
expectCmd += "https://artifacts.elastic.co/packages/oss-7.x/apt' "
expectCmd += ">/etc/apt/sources.list.d/kibana.sources && "
expectCmd += "https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/deb/debian' " # noqa: E501
expectCmd += ">/etc/apt/sources.list.d/rabbitmq.sources && "
expectCmd += "echo 'Components: main' "
expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && "
expectCmd += ">>/etc/apt/sources.list.d/rabbitmq.sources && "
expectCmd += "echo 'Types: deb' "
expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && "
expectCmd += "echo 'Suites: stable' "
expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && "
expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/elasticsearch.asc' "
expectCmd += ">>/etc/apt/sources.list.d/kibana.sources"
expectCmd += ">>/etc/apt/sources.list.d/rabbitmq.sources && "
expectCmd += "echo 'Suites: bullseye' "
expectCmd += ">>/etc/apt/sources.list.d/rabbitmq.sources && "
expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/rabbitmq.gpg' "
expectCmd += ">>/etc/apt/sources.list.d/rabbitmq.sources"
self.assertEqual(expectCmd, result)

View File

@ -0,0 +1,6 @@
---
upgrade:
- |
``elasticsearch``, ``kibana`` and ``logstash`` images have been dropped.
Zed release brings in support for ``opensearch`` and
``opensearch-dashboards`` but there's no equivalent for ``logstash``.