From ef8d7a27ab5b7d6fce8a060b1bc2709b0d1bbb98 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Fri, 9 Aug 2019 14:34:52 +0200 Subject: [PATCH] Change fluentd to td-agent on CentOS Fluentd packages in CentOS Opstools SIG repos is very old, because of old-ish ruby (2.0) packaged with CentOS 7. On the other hand - TreasureData packages for fluentd are packaged with embedded ruby as dependencies - so this change moves to those. Thanks to that we are able to move from fluentd 0.12 to 1.4.2. Still we need ruby-devel for non-td-agent distros, because we download monasca output plugin as source and build it during installation. * Bump up td.repo to version 3 * Merge sudoers into one file * Change fluentd plugin install macro to reflect changes Depends-On: https://review.opendev.org/679209/ Depends-On: https://review.opendev.org/679885/ Change-Id: I40a81103317ac044d9311b2505593056cf313bd2 --- docker/base/td.repo | 2 +- docker/fluentd/Dockerfile.j2 | 108 ++++++++++++++--------------- docker/fluentd/extend_start.sh | 4 +- docker/fluentd/fluentd_sudoers | 3 + docker/fluentd/rpm_fluentd_sudoers | 4 -- docker/macros.j2 | 10 +-- 6 files changed, 63 insertions(+), 68 deletions(-) delete mode 100644 docker/fluentd/rpm_fluentd_sudoers diff --git a/docker/base/td.repo b/docker/base/td.repo index abecf7e2c2..f4ccce31ec 100644 --- a/docker/base/td.repo +++ b/docker/base/td.repo @@ -1,5 +1,5 @@ [treasuredata] name=TreasureData -baseurl=http://packages.treasuredata.com/2/redhat/\$releasever/\$basearch +baseurl=http://packages.treasuredata.com/3/redhat/\$releasever/\$basearch gpgcheck=1 gpgkey=https://packages.treasuredata.com/GPG-KEY-td-agent diff --git a/docker/fluentd/Dockerfile.j2 b/docker/fluentd/Dockerfile.j2 index 3423bd0b2a..79397e071f 100644 --- a/docker/fluentd/Dockerfile.j2 +++ b/docker/fluentd/Dockerfile.j2 @@ -1,108 +1,108 @@ FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }} LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}" -{% if base_package_type == 'rpm' or base_arch != 'x86_64' %} +{% if base_package_type == 'rpm' and base_arch != 'x86_64' %} LABEL fluentd_version="0.12" fluentd_binary="fluentd" + {% set fluentd_user = 'fluentd' %} +{% elif base_distro == 'debian' or (base_distro == 'ubuntu' and base_arch != 'x86_64') %} +LABEL fluentd_version="0.14" fluentd_binary="fluentd" + {% set fluentd_user = 'fluentd' %} {% else %} LABEL fluentd_version="0.14" fluentd_binary="td-agent" + {% set fluentd_user = 'td-agent' %} {% endif %} {% block fluentd_header %}{% endblock %} {% import "macros.j2" as macros with context %} -{% set fluentd_user = 'fluentd' %} - {% if base_package_type == 'rpm' %} - {{ macros.configure_user(name='fluentd', groups='mysql') }} -# TODO:remove hostname package as it should be installed by the -# fluentd dependencies. -# hostname package is installed by default on CentOS/RHEL but not on -# oraclelinux. - {% set fluentd_packages = [ - 'fluentd', 'gcc-c++', - 'hostname', - 'make', - 'ruby-devel', - 'rubygem-fluent-plugin-elasticsearch', - 'rubygem-fluent-plugin-grok-parser', - 'rubygem-fluent-plugin-rewrite-tag-filter', - 'rubygem-fluent-plugin-secure-forward' + 'make' ] %} -{% elif base_package_type == 'deb' %} - {% set fluentd_packages = [ - 'g++', - 'make', - 'ruby', - 'ruby-dev', - ] %} - {% if base_arch == 'x86_64' and base_distro == 'ubuntu' %} - {% set fluentd_user = 'td-agent' %} - {{ macros.configure_user(name='td-agent', groups='mysql') }} + {% if base_arch == 'x86_64' %} {% set fluentd_packages = fluentd_packages + [ 'td-agent' ] %} {% else %} - {{ macros.configure_user(name='fluentd', groups='mysql') }} + {% set fluentd_packages = fluentd_packages + [ + 'fluentd', + 'ruby-devel', + 'rubygem-fluent-plugin-elasticsearch', + 'rubygem-fluent-plugin-grok-parser', + 'rubygem-fluent-plugin-rewrite-tag-filter' + ] %} + {% endif %} + +{% elif base_package_type == 'deb' %} + + {% set fluentd_packages = [ + 'g++', + 'make' + ] %} + {% if base_arch == 'x86_64' and base_distro == 'ubuntu' %} + {% set fluentd_packages = fluentd_packages + [ + 'td-agent' + ] %} + {% else %} + {% set fluentd_packages = fluentd_packages + [ + 'ruby', + 'ruby-dev' + ] %} {% endif %} {% endif %} +{{ macros.configure_user(name=fluentd_user, groups='mysql') }} {{ macros.install_packages(fluentd_packages | customizable("packages")) }} # Distro specific files and operations -{% if base_package_type == 'rpm' %} -COPY rpm_fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers -RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \ - && mkdir -p /var/run/fluentd \ - && chown -R fluentd: /etc/fluentd /var/run/fluentd -{% elif base_package_type == 'deb' %} - {% if base_arch == 'x86_64' and base_distro == 'ubuntu' %} -COPY fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers -RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \ - && sed -i -e "s/USER=td-agent/USER=root/" -e "s/GROUP=td-agent/GROUP=root/" /etc/init.d/td-agent \ - && rm -f /etc/td-agent/td-agent.conf \ - && chown -R td-agent: /etc/td-agent - {% else %} +# Fluentd on CentOS non x86_64 is installed from rpm +{% if base_distro == 'debian' or (base_distro == 'ubuntu' and base_arch != 'x86_64') %} RUN /usr/bin/gem install fluentd --no-rdoc --no-ri -COPY rpm_fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers -RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \ - && mkdir -p /etc/fluentd \ - && mkdir -p /var/run/fluentd \ - && chown -R fluentd: /etc/fluentd /var/run/fluentd - {% endif %} {% endif %} +COPY fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers + +RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \ + && mkdir -p /etc/{{ fluentd_user }} \ + && mkdir -p /var/run/{{ fluentd_user }} \ + && chown -R {{ fluentd_user }}: /etc/{{ fluentd_user }} /var/run/{{ fluentd_user }} COPY extend_start.sh /usr/local/bin/kolla_extend_start RUN chmod 755 /usr/local/bin/kolla_extend_start {% block fluentd_plugins_install %} -{% if base_package_type == 'deb' %} +{% if (base_package_type == 'rpm' and base_arch == 'x86_64') or base_package_type == 'deb' %} {% set fluentd_plugins = [ 'fluent-plugin-elasticsearch', 'fluent-plugin-grep', - 'fluent-plugin-grok-parser:2.1.4', + 'fluent-plugin-grok-parser', 'fluent-plugin-parser', - 'fluent-plugin-rewrite-tag-filter:2.0.0', - 'fluent-plugin-secure-forward', + 'fluent-plugin-rewrite-tag-filter', ] %} {% endif %} {{ macros.install_fluent_plugins(fluentd_plugins | customizable("plugins")) }} +{% endblock %} + +{% block fluentd_monasca_plugin_install %} # Build and install Fluentd output plugin for Monasca Log API ARG monasca_output_plugin_tag=0.1.1 ARG monasca_output_plugin_url=https://github.com/monasca/fluentd-monasca/archive/$monasca_output_plugin_tag.tar.gz -ADD $monasca_output_plugin_url /tmp/fluentd-monasca.tar.gz -RUN tar -xvf /tmp/fluentd-monasca.tar.gz -C /tmp \ + +RUN curl -sSL $monasca_output_plugin_url -o /tmp/fluentd-monasca.tar.gz \ + && tar -xvf /tmp/fluentd-monasca.tar.gz -C /tmp \ && cd /tmp/fluentd-monasca-* \ +{% if fluentd_user == 'fluentd' %} && gem build fluentd-monasca-output.gemspec \ - && gem install fluentd-monasca-output-*.gem \ +{% else %} + && td-agent-gem build fluentd-monasca-output.gemspec \ +{% endif %} && {{ macros.install_fluent_plugins(['fluentd-monasca-output-*.gem'], chain=True) }} \ && rm -rf /tmp/fluentd* diff --git a/docker/fluentd/extend_start.sh b/docker/fluentd/extend_start.sh index 93996766b0..73f3d145e8 100644 --- a/docker/fluentd/extend_start.sh +++ b/docker/fluentd/extend_start.sh @@ -6,7 +6,7 @@ # Also set the setgid permission on the /var/log/kolla directory so that new # files and sub-directories in that directory inherit its group id ("kolla"). -if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu && "${KOLLA_BASE_ARCH}" == "x86_64" ]]; then +if [[ "${KOLLA_BASE_ARCH}" == "x86_64" && "${KOLLA_BASE_DISTRO}" != "debian" ]]; then USERGROUP="td-agent:kolla" FLUENTD="td-agent" else @@ -23,6 +23,6 @@ fi if [[ $(stat -c %a /var/log/kolla) != "2775" ]]; then sudo chmod 2775 /var/log/kolla fi -if [[ $(stat -c %U:%G /var/lib/${FLUENTD}) != "${USERGROUP}" ]]; then +if [[ (-d /var/lib/${FLUENTD}) && ($(stat -c %U:%G /var/lib/${FLUENTD}) != "${USERGROUP}") ]]; then sudo chown ${USERGROUP} /var/lib/${FLUENTD} fi diff --git a/docker/fluentd/fluentd_sudoers b/docker/fluentd/fluentd_sudoers index 0111a26c48..7ff79e7f4e 100644 --- a/docker/fluentd/fluentd_sudoers +++ b/docker/fluentd/fluentd_sudoers @@ -1,4 +1,7 @@ +%kolla ALL=(root) NOPASSWD: /bin/chown fluentd\:kolla /var/log/kolla, /usr/bin/chown fluentd\:kolla /var/log/kolla %kolla ALL=(root) NOPASSWD: /bin/chown td-agent\:kolla /var/log/kolla, /usr/bin/chown td-agent\:kolla /var/log/kolla +%kolla ALL=(root) NOPASSWD: /bin/chown fluentd\:kolla /var/lib/fluentd, /usr/bin/chown fluentd\:kolla /var/lib/fluentd %kolla ALL=(root) NOPASSWD: /bin/chown td-agent\:kolla /var/lib/td-agent, /usr/bin/chown td-agent\:kolla /var/lib/td-agent %kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/log/kolla, /usr/bin/chmod 2775 /var/log/kolla +%kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/lib/fluentd, /usr/bin/chmod 2775 /var/lib/fluentd %kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/lib/td-agent, /usr/bin/chmod 2775 /var/lib/td-agent diff --git a/docker/fluentd/rpm_fluentd_sudoers b/docker/fluentd/rpm_fluentd_sudoers deleted file mode 100644 index b6a50f563f..0000000000 --- a/docker/fluentd/rpm_fluentd_sudoers +++ /dev/null @@ -1,4 +0,0 @@ -%kolla ALL=(root) NOPASSWD: /bin/chown fluentd\:kolla /var/log/kolla, /usr/bin/chown fluentd\:kolla /var/log/kolla -%kolla ALL=(root) NOPASSWD: /bin/chown fluentd\:kolla /var/lib/fluentd, /usr/bin/chown fluentd\:kolla /var/lib/fluentd -%kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/log/kolla, /usr/bin/chmod 2775 /var/log/kolla -%kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/lib/fluentd, /usr/bin/chmod 2775 /var/lib/fluentd diff --git a/docker/macros.j2 b/docker/macros.j2 index 50adf8bb64..029e995c74 100644 --- a/docker/macros.j2 +++ b/docker/macros.j2 @@ -108,14 +108,10 @@ RUN apt-get update \ {% macro install_fluent_plugins(plugins, chain=False) -%} {% if plugins is defined and plugins|length > 0 -%} {% if not chain -%} RUN {% endif -%} - {%- if base_distro in ['centos', 'oraclelinux', 'rhel'] -%} + {%- if base_arch == 'x86_64' and base_distro in ['centos', 'oraclelinux', 'rhel', 'ubuntu'] -%} + ulimit -n 65536 && td-agent-gem install {{ plugins | join(' ') }} + {%- else -%} ulimit -n 65536 && gem install --minimal-deps {{ plugins | join(' ') }} - {%- elif base_distro in ['debian', 'ubuntu'] -%} - {%- if base_arch == 'x86_64' and base_distro == 'ubuntu' -%} - ulimit -n 65536 && td-agent-gem install {{ plugins | join(' ') }} - {%- else -%} - ulimit -n 65536 && gem install --minimal-deps {{ plugins | join(' ') }} - {%- endif %} {%- endif %} {%- endif %} {%- endmacro %}