openstack/kolla
Code Issues Proposed changes

Change fluentd to td-agent on CentOS

Browse Source 
Fluentd packages in CentOS Opstools SIG repos is very old,
because of old-ish ruby (2.0) packaged with CentOS 7.

On the other hand - TreasureData packages for fluentd are packaged
with embedded ruby as dependencies - so this change moves to those.
Thanks to that we are able to move from fluentd 0.12 to 1.4.2.

Still we need ruby-devel for non-td-agent distros,
because we download monasca output plugin as source and build it
during installation.

* Bump up td.repo to version 3
* Merge sudoers into one file
* Change fluentd plugin install macro to reflect changes

Depends-On: https://review.opendev.org/679209/
Depends-On: https://review.opendev.org/679885/
Change-Id: I40a81103317ac044d9311b2505593056cf313bd2
This commit is contained in:
Michal Nasiadka 2019-08-09 14:34:52 +02:00
parent b602f0144a
commit ef8d7a27ab
6 changed files with 63 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docker/base/td.repo
View File

@ -1,5 +1,5 @@
[treasuredata]
name=TreasureData
baseurl=http://packages.treasuredata.com/2/redhat/\$releasever/\$basearch
baseurl=http://packages.treasuredata.com/3/redhat/\$releasever/\$basearch
gpgcheck=1
gpgkey=https://packages.treasuredata.com/GPG-KEY-td-agent

108
docker/fluentd/Dockerfile.j2
View File

@ -1,108 +1,108 @@
FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }}
LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
{% if base_package_type == 'rpm' or base_arch != 'x86_64' %}
{% if base_package_type == 'rpm' and base_arch != 'x86_64' %}
LABEL fluentd_version="0.12" fluentd_binary="fluentd"
{% set fluentd_user = 'fluentd' %}
{% elif base_distro == 'debian' or (base_distro == 'ubuntu' and base_arch != 'x86_64') %}
LABEL fluentd_version="0.14" fluentd_binary="fluentd"
{% set fluentd_user = 'fluentd' %}
{% else %}
LABEL fluentd_version="0.14" fluentd_binary="td-agent"
{% set fluentd_user = 'td-agent' %}
{% endif %}
{% block fluentd_header %}{% endblock %}
{% import "macros.j2" as macros with context %}
{% set fluentd_user = 'fluentd' %}
{% if base_package_type == 'rpm' %}
{{ macros.configure_user(name='fluentd', groups='mysql') }}
# TODO:remove hostname package as it should be installed by the
# fluentd dependencies.
# hostname package is installed by default on CentOS/RHEL but not on
# oraclelinux.
{% set fluentd_packages = [
'fluentd',
'gcc-c++',
'hostname',
'make',
'ruby-devel',
'rubygem-fluent-plugin-elasticsearch',
'rubygem-fluent-plugin-grok-parser',
'rubygem-fluent-plugin-rewrite-tag-filter',
'rubygem-fluent-plugin-secure-forward'
'make'
] %}
{% elif base_package_type == 'deb' %}
{% set fluentd_packages = [
'g++',
'make',
'ruby',
'ruby-dev',
] %}
{% if base_arch == 'x86_64' and base_distro == 'ubuntu' %}
{% set fluentd_user = 'td-agent' %}
{{ macros.configure_user(name='td-agent', groups='mysql') }}
{% if base_arch == 'x86_64' %}
{% set fluentd_packages = fluentd_packages + [
'td-agent'
] %}
{% else %}
{{ macros.configure_user(name='fluentd', groups='mysql') }}
{% set fluentd_packages = fluentd_packages + [
'fluentd',
'ruby-devel',
'rubygem-fluent-plugin-elasticsearch',
'rubygem-fluent-plugin-grok-parser',
'rubygem-fluent-plugin-rewrite-tag-filter'
] %}
{% endif %}
{% elif base_package_type == 'deb' %}
{% set fluentd_packages = [
'g++',
'make'
] %}
{% if base_arch == 'x86_64' and base_distro == 'ubuntu' %}
{% set fluentd_packages = fluentd_packages + [
'td-agent'
] %}
{% else %}
{% set fluentd_packages = fluentd_packages + [
'ruby',
'ruby-dev'
] %}
{% endif %}
{% endif %}
{{ macros.configure_user(name=fluentd_user, groups='mysql') }}
{{ macros.install_packages(fluentd_packages | customizable("packages")) }}
# Distro specific files and operations
{% if base_package_type == 'rpm' %}
COPY rpm_fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers
RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \
&& mkdir -p /var/run/fluentd \
&& chown -R fluentd: /etc/fluentd /var/run/fluentd
{% elif base_package_type == 'deb' %}
{% if base_arch == 'x86_64' and base_distro == 'ubuntu' %}
COPY fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers
RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \
&& sed -i -e "s/USER=td-agent/USER=root/" -e "s/GROUP=td-agent/GROUP=root/" /etc/init.d/td-agent \
&& rm -f /etc/td-agent/td-agent.conf \
&& chown -R td-agent: /etc/td-agent
{% else %}
# Fluentd on CentOS non x86_64 is installed from rpm
{% if base_distro == 'debian' or (base_distro == 'ubuntu' and base_arch != 'x86_64') %}
RUN /usr/bin/gem install fluentd --no-rdoc --no-ri
COPY rpm_fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers
RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \
&& mkdir -p /etc/fluentd \
&& mkdir -p /var/run/fluentd \
&& chown -R fluentd: /etc/fluentd /var/run/fluentd
{% endif %}
{% endif %}
COPY fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers
RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \
&& mkdir -p /etc/{{ fluentd_user }} \
&& mkdir -p /var/run/{{ fluentd_user }} \
&& chown -R {{ fluentd_user }}: /etc/{{ fluentd_user }} /var/run/{{ fluentd_user }}
COPY extend_start.sh /usr/local/bin/kolla_extend_start
RUN chmod 755 /usr/local/bin/kolla_extend_start
{% block fluentd_plugins_install %}
{% if base_package_type == 'deb' %}
{% if (base_package_type == 'rpm' and base_arch == 'x86_64') or base_package_type == 'deb' %}
{% set fluentd_plugins = [
'fluent-plugin-elasticsearch',
'fluent-plugin-grep',
'fluent-plugin-grok-parser:2.1.4',
'fluent-plugin-grok-parser',
'fluent-plugin-parser',
'fluent-plugin-rewrite-tag-filter:2.0.0',
'fluent-plugin-secure-forward',
'fluent-plugin-rewrite-tag-filter',
] %}
{% endif %}
{{ macros.install_fluent_plugins(fluentd_plugins | customizable("plugins")) }}
{% endblock %}
{% block fluentd_monasca_plugin_install %}
# Build and install Fluentd output plugin for Monasca Log API
ARG monasca_output_plugin_tag=0.1.1
ARG monasca_output_plugin_url=https://github.com/monasca/fluentd-monasca/archive/$monasca_output_plugin_tag.tar.gz
ADD $monasca_output_plugin_url /tmp/fluentd-monasca.tar.gz
RUN tar -xvf /tmp/fluentd-monasca.tar.gz -C /tmp \
RUN curl -sSL $monasca_output_plugin_url -o /tmp/fluentd-monasca.tar.gz \
&& tar -xvf /tmp/fluentd-monasca.tar.gz -C /tmp \
&& cd /tmp/fluentd-monasca-* \
{% if fluentd_user == 'fluentd' %}
&& gem build fluentd-monasca-output.gemspec \
&& gem install fluentd-monasca-output-*.gem \
{% else %}
&& td-agent-gem build fluentd-monasca-output.gemspec \
{% endif %}
&& {{ macros.install_fluent_plugins(['fluentd-monasca-output-*.gem'], chain=True) }} \
&& rm -rf /tmp/fluentd*

4
docker/fluentd/extend_start.sh
View File

@ -6,7 +6,7 @@
# Also set the setgid permission on the /var/log/kolla directory so that new
# files and sub-directories in that directory inherit its group id ("kolla").
if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu && "${KOLLA_BASE_ARCH}" == "x86_64" ]]; then
if [[ "${KOLLA_BASE_ARCH}" == "x86_64" && "${KOLLA_BASE_DISTRO}" != "debian" ]]; then
USERGROUP="td-agent:kolla"
FLUENTD="td-agent"
else
@ -23,6 +23,6 @@ fi
if [[ $(stat -c %a /var/log/kolla) != "2775" ]]; then
sudo chmod 2775 /var/log/kolla
fi
if [[ $(stat -c %U:%G /var/lib/${FLUENTD}) != "${USERGROUP}" ]]; then
if [[ (-d /var/lib/${FLUENTD}) && ($(stat -c %U:%G /var/lib/${FLUENTD}) != "${USERGROUP}") ]]; then
sudo chown ${USERGROUP} /var/lib/${FLUENTD}
fi

3
docker/fluentd/fluentd_sudoers
View File

@ -1,4 +1,7 @@
%kolla ALL=(root) NOPASSWD: /bin/chown fluentd\:kolla /var/log/kolla, /usr/bin/chown fluentd\:kolla /var/log/kolla
%kolla ALL=(root) NOPASSWD: /bin/chown td-agent\:kolla /var/log/kolla, /usr/bin/chown td-agent\:kolla /var/log/kolla
%kolla ALL=(root) NOPASSWD: /bin/chown fluentd\:kolla /var/lib/fluentd, /usr/bin/chown fluentd\:kolla /var/lib/fluentd
%kolla ALL=(root) NOPASSWD: /bin/chown td-agent\:kolla /var/lib/td-agent, /usr/bin/chown td-agent\:kolla /var/lib/td-agent
%kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/log/kolla, /usr/bin/chmod 2775 /var/log/kolla
%kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/lib/fluentd, /usr/bin/chmod 2775 /var/lib/fluentd
%kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/lib/td-agent, /usr/bin/chmod 2775 /var/lib/td-agent

4
docker/fluentd/rpm_fluentd_sudoers
View File

@ -1,4 +0,0 @@
%kolla ALL=(root) NOPASSWD: /bin/chown fluentd\:kolla /var/log/kolla, /usr/bin/chown fluentd\:kolla /var/log/kolla
%kolla ALL=(root) NOPASSWD: /bin/chown fluentd\:kolla /var/lib/fluentd, /usr/bin/chown fluentd\:kolla /var/lib/fluentd
%kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/log/kolla, /usr/bin/chmod 2775 /var/log/kolla
%kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/lib/fluentd, /usr/bin/chmod 2775 /var/lib/fluentd

10
docker/macros.j2
View File

@ -108,14 +108,10 @@ RUN apt-get update \
{% macro install_fluent_plugins(plugins, chain=False) -%}
{% if plugins is defined and plugins|length > 0 -%}
{% if not chain -%} RUN {% endif -%}
{%- if base_distro in ['centos', 'oraclelinux', 'rhel'] -%}
{%- if base_arch == 'x86_64' and base_distro in ['centos', 'oraclelinux', 'rhel', 'ubuntu'] -%}
ulimit -n 65536 && td-agent-gem install {{ plugins | join(' ') }}
{%- else -%}
ulimit -n 65536 && gem install --minimal-deps {{ plugins | join(' ') }}
{%- elif base_distro in ['debian', 'ubuntu'] -%}
{%- if base_arch == 'x86_64' and base_distro == 'ubuntu' -%}
ulimit -n 65536 && td-agent-gem install {{ plugins | join(' ') }}
{%- else -%}
ulimit -n 65536 && gem install --minimal-deps {{ plugins | join(' ') }}
{%- endif %}
{%- endif %}
{%- endif %}
{%- endmacro %}